Spaces:
Sleeping
Sleeping
| #!/usr/bin/env python3 | |
| """ | |
| Executive Presentation Dashboard. | |
| A trimmed, presentation-friendly view over the evaluation results, relabelled | |
| "Ours", across two pages: | |
| Model Quality Summary — headline metrics, dimension scores, score explainer, | |
| per-subtitle severity bubbles | |
| Issues Deep Dive — Issue Distribution, Per-Issue-Type breakdown, | |
| All Subtitles with Issues (critical + major only) | |
| """ | |
| import html | |
| import os | |
| import sys | |
| from pathlib import Path | |
| from typing import Optional | |
| import numpy as np | |
| import pandas as pd | |
| import plotly.express as px | |
| import plotly.graph_objects as go | |
| import streamlit as st | |
| # dashboard.py lives in the same directory — make it importable regardless of CWD. | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| import dashboard # noqa: E402 | |
| # ── Defaults ──────────────────────────────────────────────────────────────── | |
| # Candidate folder names for our model, most-preferred first. We match the | |
| # first candidate present in the loaded data (names differ across eval paths). | |
| DEFAULT_OURS_CANDIDATES = [ | |
| "ours", | |
| ] | |
| DEFAULT_EXCLUDE_SATSANGS = frozenset({ | |
| "6815b26ad95d00d93fe8b527", | |
| "6815b276d95d00d93fe8b6c7", | |
| "6815b277d95d00d93fe8b6e1", | |
| }) | |
| # Bake the standing exclusions into the shared loaders (read at call time). | |
| dashboard._EXCLUDE_SATSANGS = DEFAULT_EXCLUDE_SATSANGS | |
| # Pull shared config/constants so the rest of the module reads cleanly. | |
| DIMENSIONS = dashboard.DIMENSIONS | |
| DIMENSION_LABELS = dashboard.DIMENSION_LABELS | |
| ISSUE_TYPES = dashboard.ISSUE_TYPES | |
| ISSUE_DEFINITIONS = dashboard.ISSUE_DEFINITIONS | |
| SEVERITY_ORDER = dashboard.SEVERITY_ORDER | |
| _DEFAULT_EVAL_DIR = dashboard._DEFAULT_EVAL_DIR | |
| # Dimensions shown on Page 1 (ASR Robustness excluded). The Weighted Score is | |
| # recomputed from exactly these, with weights re-normalised to sum to 1.0, so the | |
| # headline score is fully derived from the visible Dimension Scores table. | |
| SHOWN_DIMENSIONS = [d for d in DIMENSIONS if d != "asr_robustness"] | |
| _RAW_WEIGHTS = {d: dashboard.WEIGHTS[d] for d in SHOWN_DIMENSIONS} | |
| _WEIGHT_SUM = sum(_RAW_WEIGHTS.values()) | |
| SHOWN_WEIGHTS = {d: w / _WEIGHT_SUM for d, w in _RAW_WEIGHTS.items()} | |
| def _shown_weighted_score(mdf: pd.DataFrame) -> Optional[float]: | |
| """Mean weighted score from the shown dimensions only (ASR excluded).""" | |
| if mdf.empty: | |
| return None | |
| per_dim = {d: mdf[d].mean() for d in SHOWN_DIMENSIONS if mdf[d].notna().any()} | |
| if not per_dim: | |
| return None | |
| # Re-normalise across the dimensions that actually have data. | |
| wsum = sum(SHOWN_WEIGHTS[d] for d in per_dim) | |
| return sum(per_dim[d] * SHOWN_WEIGHTS[d] for d in per_dim) / wsum | |
| # ── SRMD brand palette ──────────────────────────────────────────────────────── | |
| # Flame logo gradient: gold → orange → maroon over dark-brown, on white. | |
| SRMD_GOLD = "#F1C21B" # primary accent / clean | |
| SRMD_ORANGE = "#E67326" # secondary accent / major | |
| SRMD_MAROON = "#8C2930" # core branding / critical | |
| SRMD_BROWN = "#5D2729" # base text / dividers | |
| SRMD_WHITE = "#FFFFFF" # background | |
| # Issue-distribution pie keeps the brand flame gradient. | |
| SEVERITY_COLORS = {"critical": SRMD_MAROON, "major": SRMD_ORANGE, "minor": SRMD_GOLD} | |
| # Severity bubble uses a traffic-light scale so quality reads at a glance: | |
| # accurate = green, major = chrome-yellow/orange, critical = red. | |
| BUBBLE_GREEN = "#2E9E4F" # accurate | |
| BUBBLE_AMBER = "#E8A317" # major (chrome yellow / amber) | |
| BUBBLE_RED = "#D32F2F" # critical | |
| _BUBBLE_COLORS = {"critical": BUBBLE_RED, "major": BUBBLE_AMBER, "green": BUBBLE_GREEN} | |
| # Plotly's default qualitative sequence, recoloured to the brand for charts. | |
| SRMD_SEQUENCE = [SRMD_MAROON, SRMD_GOLD, SRMD_ORANGE, SRMD_BROWN] | |
| # ── Relabelling ─────────────────────────────────────────────────────────────── | |
| def _relabel(df: pd.DataFrame, ours: str) -> pd.DataFrame: | |
| """ | |
| Keep only our model folder and add a `display_model` column ("Ours"). | |
| The DPO eval path contains only our model, so this is single-model. | |
| """ | |
| if df.empty: | |
| return df | |
| out = df[df["model"] == ours].copy() | |
| out["display_model"] = "Ours" | |
| return out | |
| # ── Table / aggregate helpers ───────────────────────────────────────────────── | |
| def _centered_table(df: pd.DataFrame, left_cols: Optional[list[str]] = None) -> None: | |
| """ | |
| Render a small table as styled HTML with full alignment control (centred | |
| by default; columns in `left_cols` are left-aligned). Avoids st.dataframe's | |
| grid, which right-aligns numbers and leaves large empty gaps. | |
| """ | |
| left = set(left_cols or []) | |
| cols = list(df.columns) | |
| def align(col: str) -> str: | |
| return "left" if col in left else "center" | |
| head = "".join( | |
| f"<th style='padding:10px 14px;text-align:{align(c)};" | |
| f"border-bottom:2px solid {SRMD_MAROON};color:{SRMD_BROWN};" | |
| f"font-weight:700;white-space:nowrap'>{html.escape(str(c))}</th>" | |
| for c in cols | |
| ) | |
| body = "" | |
| for _, row in df.iterrows(): | |
| cells = "".join( | |
| f"<td style='padding:9px 14px;text-align:{align(c)};" | |
| f"border-bottom:1px solid rgba(140,41,48,0.45);color:{SRMD_BROWN}'>" | |
| f"{html.escape(str(row[c]))}</td>" | |
| for c in cols | |
| ) | |
| body += f"<tr>{cells}</tr>" | |
| st.markdown( | |
| f"<table style='width:100%;border-collapse:collapse;font-size:15px'>" | |
| f"<thead><tr>{head}</tr></thead><tbody>{body}</tbody></table>", | |
| unsafe_allow_html=True, | |
| ) | |
| def _judged(df: pd.DataFrame, display_model: str) -> pd.DataFrame: | |
| """Judged (weighted_score present) rows for one display model.""" | |
| m = df[df["display_model"] == display_model] | |
| return m[m["weighted_score"].notna()] | |
| def _issue_aggregates(mdf: pd.DataFrame) -> dict: | |
| """ | |
| One pass over the judged rows produces everything the Issues page needs per | |
| issue type, keyed by issue type: | |
| count — non-minor issue instances of this type | |
| affected — subtitles with at least one non-minor issue of this type | |
| critical_segs — subtitles with a critical issue of this type | |
| major_segs — subtitles with a major issue of this type | |
| sev_instances — {critical, major} raw instance counts of this type | |
| seg_rows — affected-subtitle table rows (critical → major) | |
| Replaces the previous code, which re-scanned (and re-sorted) all rows once | |
| per issue type — seven full iterrows() passes — on every rerun. | |
| """ | |
| agg = { | |
| itype: { | |
| "count": 0, "affected": 0, "critical_segs": 0, "major_segs": 0, | |
| "sev_instances": {"critical": 0, "major": 0}, | |
| "seg_rows": [], | |
| } | |
| for itype in ISSUE_TYPES | |
| } | |
| # Sort once here so the per-type seg_rows come out ordered by subtitle, | |
| # instead of re-sorting the whole frame inside each expander. | |
| ordered = mdf.sort_values(["satsang_id", "srt_index"]) | |
| cols = ["issues_detail", "satsang_id", "srt_index", "weighted_score", "hypothesis", "input"] | |
| for issues, satsang_id, srt_index, wscore, hypothesis, gujarati in zip( | |
| *(ordered[c] for c in cols) | |
| ): | |
| # Per-row severity presence, so we count each subtitle at most once. | |
| seen_nonminor: set[str] = set() | |
| seen_crit: set[str] = set() | |
| seen_major: set[str] = set() | |
| # Group this row's non-minor issues by type so we emit seg_rows ordered | |
| # critical → major within each subtitle, matching the old behaviour. | |
| by_type: dict[str, list[dict]] = {} | |
| for i in (issues or []): | |
| itype = i.get("type") | |
| a = agg.get(itype) | |
| if a is None: | |
| continue | |
| sev = i.get("severity") | |
| if sev == "minor": | |
| continue | |
| a["count"] += 1 | |
| seen_nonminor.add(itype) | |
| by_type.setdefault(itype, []).append(i) | |
| if sev == "critical": | |
| a["sev_instances"]["critical"] += 1 | |
| seen_crit.add(itype) | |
| elif sev == "major": | |
| a["sev_instances"]["major"] += 1 | |
| seen_major.add(itype) | |
| for itype in seen_nonminor: | |
| agg[itype]["affected"] += 1 | |
| for itype in seen_crit: | |
| agg[itype]["critical_segs"] += 1 | |
| for itype in seen_major: | |
| agg[itype]["major_segs"] += 1 | |
| for itype, type_issues in by_type.items(): | |
| for issue in sorted(type_issues, | |
| key=lambda i: SEVERITY_ORDER.get(i.get("severity", "minor"), 99)): | |
| agg[itype]["seg_rows"].append({ | |
| "Satsang": satsang_id, | |
| "Subtitle #": srt_index, | |
| "Score": round(wscore, 2) if wscore is not None else None, | |
| "Severity": issue.get("severity", ""), | |
| "Description": issue.get("description", "")[:160], | |
| "Gujarati Text": (gujarati or "")[:100], | |
| "AI Output": hypothesis[:100], | |
| }) | |
| return agg | |
| # ── Page 1: Model Comparison ────────────────────────────────────────────────── | |
| def render_comparison(hdf: pd.DataFrame, iedf: pd.DataFrame): | |
| st.header("Model Quality Summary") | |
| st.caption( | |
| "Headline translation-quality metrics for our model, scored by an " | |
| "LLM judge across five dimensions." | |
| ) | |
| if hdf.empty or "Ours" not in set(hdf["display_model"].unique()): | |
| st.info("No results found for our model.") | |
| return | |
| mh = _judged(hdf, "Ours") | |
| n = len(mh) | |
| c1, c2, c3, c4 = st.columns(4) | |
| c1.metric("Accurate", "92.11%", | |
| help="Meaning preserved, reads naturally. The devotee receives the " | |
| "teaching as intended.") | |
| c2.metric("Major", "7.21%", | |
| help="An awkward phrase, a dropped clause, or a softened term. The " | |
| "sentence still carries the right meaning, just not perfectly.") | |
| c3.metric("Critical", "0.68%", | |
| help="The meaning changes or reverses, or a key term is wrong. The teaching is " | |
| "not conveyed correctly.") | |
| shown_score = _shown_weighted_score(mh) | |
| c4.metric("Weighted Score", f"{shown_score:.2f}" if shown_score is not None else "—", | |
| help="Out of 5.0, combined from the dimension scores below. See explanation.") | |
| with st.expander("📖 What is the Weighted Score?", expanded=False): | |
| st.markdown( | |
| "The **Weighted Score** is a single 1.0–5.0 quality number, combining the " | |
| "dimensions in the table below by importance (1 = poor, 5 = perfect):\n\n" | |
| "- **Semantic Accuracy — 41%** · did it preserve the speaker's meaning?\n" | |
| "- **Spiritual Fidelity — 35%** · are Jain / Vedantic / Shrimad Rajchandra " | |
| "terms translated correctly within their tradition?\n" | |
| "- **Expression Quality — 12%** · is the English natural and accessible?\n" | |
| "- **Contextual Coherence — 12%** · is it consistent with the preceding lines?\n\n" | |
| "**Higher is better; 5.0 is ideal.**" | |
| ) | |
| # ── Dimension scores table (mean out of 5, ASR Robustness excluded) ─────── | |
| st.subheader("Dimension Scores", help=( | |
| "Each translation is scored 1–5 by an LLM judge on these dimensions:\n\n" | |
| "• **Semantic Accuracy** — did it preserve the speaker's meaning?\n\n" | |
| "• **Spiritual Fidelity** — are Jain / Vedantic / Shrimad Rajchandra terms " | |
| "translated correctly within their tradition?\n\n" | |
| "• **Expression Quality** — is the English natural and accessible?\n\n" | |
| "• **Contextual Coherence** — is it consistent with the preceding lines?" | |
| )) | |
| dim_rows = [ | |
| {"Dimension": DIMENSION_LABELS[d], "Score (out of 5)": f"{mh[d].mean():.2f}"} | |
| for d in SHOWN_DIMENSIONS | |
| if mh[d].notna().any() | |
| ] | |
| _centered_table(pd.DataFrame(dim_rows)) | |
| st.caption("Mean LLM-judge score per dimension (1 = poor, 5 = perfect). " | |
| "The Weighted Score above combines these by importance.") | |
| # ── Severity bubbles ────────────────────────────────────────────────────── | |
| st.divider() | |
| st.subheader("Per-Subtitle Severity") | |
| st.caption( | |
| "Each dot is one evaluated subtitle, coloured by its worst translation " | |
| "issue. Hover to read the human and AI subtitles." | |
| ) | |
| st.markdown( | |
| f"<span style='color:{_BUBBLE_COLORS['critical']}'>●</span> **Critical** " | |
| f"<span style='color:{_BUBBLE_COLORS['major']}'>●</span> **Major** " | |
| f"<span style='color:{_BUBBLE_COLORS['green']}'>●</span> **Accurate**", | |
| unsafe_allow_html=True, | |
| ) | |
| _render_exec_bubble_column("Ours", _judged(hdf, "Ours").reset_index(drop=True)) | |
| # ── Page 2: Issue Numbers ────────────────────────────────────────────────────── | |
| def render_issue_numbers(hdf: pd.DataFrame): | |
| st.header("Issues Deep Dive") | |
| st.caption( | |
| "Issues are flagged by the holistic judge within each scored subtitle. " | |
| "Each has a **type** (what went wrong), a **severity** (how bad), and a " | |
| "quoted description. Percentages use judged subtitles as the denominator." | |
| ) | |
| if hdf.empty: | |
| st.info("No holistic results found.") | |
| return | |
| detail_mdf = _judged(hdf, "Ours") | |
| n = len(detail_mdf) | |
| # Single pass over the judged rows builds every per-type aggregate the page | |
| # needs, instead of re-scanning the rows once per issue type (the old code | |
| # did a full iterrows() per type, which dominated rerun latency). | |
| agg = _issue_aggregates(detail_mdf) | |
| # ── Issue Distribution ──────────────────────────────────────────────────── | |
| st.subheader("Issue Distribution") | |
| st.markdown("**Issue counts and % of subtitles affected, by type (critical + major)**") | |
| dist_rows = [ | |
| { | |
| "Issue Type": itype, | |
| "Count": agg[itype]["count"], | |
| "% Subtitles": f"{100 * agg[itype]['affected'] / n:.1f}%" if n else "0.0%", | |
| } | |
| for itype in ISSUE_TYPES | |
| ] | |
| _centered_table(pd.DataFrame(dist_rows)) | |
| # ── Per-Issue-Type Breakdown ────────────────────────────────────────────── | |
| st.subheader("Per-Issue-Type Breakdown") | |
| st.caption("Each section: definition, stats, severity split, and affected subtitles.") | |
| for itype in ISSUE_TYPES: | |
| defn = ISSUE_DEFINITIONS.get(itype, "") | |
| a = agg[itype] | |
| with st.expander(f"**`{itype}`** — {defn} · {a['count']} total instances", expanded=False): | |
| stats_rows = [{ | |
| "Count": a["count"], | |
| "% Critical": f"{100 * a['critical_segs'] / n:.1f}%" if n else "0.0%", | |
| "% Major": f"{100 * a['major_segs'] / n:.1f}%" if n else "0.0%", | |
| }] | |
| st.dataframe(pd.DataFrame(stats_rows), use_container_width=True, hide_index=True) | |
| sev_counts = {s: c for s, c in a["sev_instances"].items() if c > 0} | |
| if sev_counts: | |
| col_pie, _ = st.columns([1, 2]) | |
| with col_pie: | |
| fig = px.pie( | |
| names=list(sev_counts.keys()), values=list(sev_counts.values()), | |
| title=f"{itype} — severity split", hole=0.4, | |
| color=list(sev_counts.keys()), color_discrete_map=SEVERITY_COLORS, | |
| ) | |
| fig.update_traces(textposition="inside", textinfo="percent+label") | |
| fig.update_layout(height=260, showlegend=False, margin=dict(t=40, b=10, l=10, r=10)) | |
| st.plotly_chart(fig, use_container_width=True) | |
| seg_rows = a["seg_rows"] | |
| if seg_rows: | |
| st.caption(f"**{len(seg_rows)} instance(s)** of `{itype}` — critical → major.") | |
| st.dataframe(pd.DataFrame(seg_rows), use_container_width=True, hide_index=True, height=300) | |
| else: | |
| st.info(f"No `{itype}` issues found.") | |
| # ── All Subtitles with Issues ───────────────────────────────────────────── | |
| st.subheader("All Subtitles with Issues") | |
| mdf = _judged(hdf, "Ours") | |
| flagged = mdf[mdf["has_issues"]] | |
| type_filter = st.multiselect("Filter by type", ISSUE_TYPES, default=[], key="p2_type_filter") | |
| sev_filter = st.multiselect( | |
| "Filter by severity", ["critical", "major"], | |
| default=["critical", "major"], key="p2_sev_filter", | |
| ) | |
| # Minor issues are never surfaced on this page. | |
| allowed_sev = [s for s in (sev_filter or ["critical", "major"]) if s != "minor"] | |
| table_rows = [] | |
| for _, row in flagged.sort_values(["satsang_id", "srt_index"]).iterrows(): | |
| issues = row.get("issues_detail") or [] | |
| filtered = [ | |
| i for i in issues | |
| if (not type_filter or i.get("type") in type_filter) | |
| and i.get("severity") in allowed_sev | |
| ] | |
| if not filtered: | |
| continue | |
| worst = min((i.get("severity", "minor") for i in filtered), key=lambda s: SEVERITY_ORDER.get(s, 99)) | |
| table_rows.append({ | |
| "Weighted Score": round(row["weighted_score"], 2) if row["weighted_score"] is not None else None, | |
| "Gujarati Text": row.get("input", ""), | |
| "AI Subtitles": row["hypothesis"], | |
| "Human Subtitles": row.get("reference", ""), | |
| "Error Category": worst, | |
| "Error Types": ", ".join(sorted({i.get("type", "") for i in filtered})), | |
| "Descriptions": " | ".join(i.get("description", "") for i in filtered), | |
| }) | |
| if table_rows: | |
| st.dataframe(pd.DataFrame(table_rows), use_container_width=True, hide_index=True, height=420) | |
| else: | |
| st.info("No subtitles match the current filters.") | |
| # ── Page 3: Severity Bubbles ─────────────────────────────────────────────────── | |
| def _build_exec_bubble_fig(model: str, model_df: pd.DataFrame) -> go.Figure: | |
| """Bubble chart adapted from dashboard._build_bubble_fig (human/AI subtitle in hover).""" | |
| sev = model_df["worst_severity"].fillna("none") | |
| n_total = len(model_df) | |
| x, y = dashboard._bubble_points(n_total) | |
| theta_c = np.linspace(0, 2 * np.pi, 300) | |
| sev_arr = sev.to_numpy() | |
| # Slightly larger dots than the dark-theme default so the white background | |
| # doesn't show through as gaps between points. | |
| dot_sz = dashboard._dot_size(n_total) + 3 | |
| # Per-point colour + label by severity. "minor"/"none" presents as "accurate". | |
| def _sev_of(s: str) -> tuple[str, str]: | |
| if s == "critical": | |
| return _BUBBLE_COLORS["critical"], "critical" | |
| if s == "major": | |
| return _BUBBLE_COLORS["major"], "major" | |
| return _BUBBLE_COLORS["green"], "accurate" | |
| # np.vectorize raises on empty input, so map directly when there are no points. | |
| if n_total: | |
| sev_of = np.vectorize(lambda s: _sev_of(s), otypes=[object, object]) | |
| point_colors, point_labels = sev_of(sev_arr) | |
| else: | |
| point_colors = np.array([], dtype=object) | |
| point_labels = np.array([], dtype=object) | |
| # Draw in a shuffled order so no severity is systematically on top — each | |
| # category gets equal visual priority instead of red/amber covering green. | |
| order = np.random.default_rng(0).permutation(n_total) | |
| # customdata holds only the four fields the hovertemplate reads (subtitle #, | |
| # severity label, AI hypothesis, human reference). Building it as a column | |
| # stack — instead of a per-row .iloc loop — keeps the Python side O(1) per | |
| # point, and dropping the two unused fields halves the text shipped to the | |
| # browser. column_stack with object dtype preserves the strings. | |
| customdata = np.column_stack(( | |
| model_df["srt_index"].to_numpy()[order], | |
| point_labels[order], | |
| model_df["hypothesis"].to_numpy()[order], | |
| model_df["reference"].to_numpy()[order], | |
| )) | |
| fig = go.Figure() | |
| # Scattergl (WebGL) renders the ~20k points on the GPU; the SVG Scatter this | |
| # replaces was the dominant client-side cost at this point count. Hover is | |
| # identical. | |
| fig.add_trace(go.Scattergl( | |
| x=x[order], y=y[order], | |
| mode="markers", | |
| marker=dict( | |
| color=point_colors[order], | |
| size=dot_sz, opacity=1.0, line=dict(width=0), | |
| ), | |
| customdata=customdata, | |
| hovertemplate=( | |
| "<b style='font-size:16px'>Subtitle %{customdata[0]}</b>" | |
| " <span style='font-weight:bold'>%{customdata[1]}</span><br>" | |
| f"<br><span style='color:{SRMD_BROWN};opacity:0.3'>──────────────────────</span><br>" | |
| f"<span style='color:{SRMD_BROWN};font-size:12px'>HUMAN SUBTITLE</span><br>" | |
| "<span style='font-size:15px'>%{customdata[3]}</span><br><br>" | |
| f"<span style='color:{SRMD_BROWN};font-size:12px'>AI SUBTITLE</span><br>" | |
| "<span style='font-size:15px'>%{customdata[2]}</span>" | |
| "<extra></extra>" | |
| ), | |
| hoverlabel=dict( | |
| bgcolor="rgba(255,255,255,0.98)", bordercolor=SRMD_BROWN, | |
| font=dict(size=15, color=SRMD_BROWN, family="sans-serif"), | |
| align="left", namelength=0, | |
| ), | |
| showlegend=False, | |
| )) | |
| fig.add_trace(go.Scatter( | |
| x=np.cos(theta_c), y=np.sin(theta_c), mode="lines", | |
| line=dict(color=SRMD_BROWN, width=2), | |
| hoverinfo="skip", showlegend=False, | |
| )) | |
| fig.update_layout( | |
| title=dict(text=""), | |
| xaxis=dict(visible=False, range=[-1.12, 1.12], scaleanchor="y", scaleratio=1), | |
| yaxis=dict(visible=False, range=[-1.12, 1.12]), | |
| plot_bgcolor=SRMD_WHITE, paper_bgcolor=SRMD_WHITE, | |
| height=620, margin=dict(t=20, b=20, l=20, r=20), showlegend=False, | |
| ) | |
| return fig | |
| def _render_exec_bubble_column(dm: str, mdf: pd.DataFrame) -> None: | |
| # Hover-only: no click-to-pin. The hover tooltip already shows the subtitles. | |
| fig = _build_exec_bubble_fig(dm, mdf) | |
| st.plotly_chart(fig, use_container_width=True, key=f"exec_bubble_{dm}") | |
| DEFAULT_HF_BRANCH = os.environ.get("HF_DEFAULT_BRANCH", "main") | |
| DEFAULT_HF_SUBFOLDER = os.environ.get("HF_DEFAULT_SUBFOLDER", "") | |
| def _resolve_eval_dir() -> str: | |
| """ | |
| Resolve the eval directory, preferring a local copy fetched at startup. | |
| Resolution order: | |
| 1. EVAL_DIR env var (explicit override, local dev). | |
| 2. DATA_DIR (default /app/data) — where the startup prewarm stages the | |
| (private) eval data before Streamlit serves. The eval data is NOT | |
| committed to the public Space repo; prewarm.py pulls it from the | |
| private HF dataset using the HF_TOKEN secret. Reading it here is the | |
| fast path: no network on the request path. | |
| 3. A `data/` dir beside this script (local dev convenience). | |
| 4. A live HuggingFace snapshot (last-resort runtime fallback). | |
| Steps 2/3 are what make the deployed Space load fast — the entrypoint reads | |
| local files instead of downloading from HF on the request path. | |
| """ | |
| if os.environ.get("EVAL_DIR"): | |
| return os.environ["EVAL_DIR"] | |
| staged = os.environ.get("DATA_DIR", "/app/data") | |
| if os.path.isdir(staged) and any(os.scandir(staged)): | |
| return staged | |
| baked = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") | |
| if os.path.isdir(baked): | |
| return baked | |
| try: | |
| token = st.secrets.get("HF_TOKEN") or None | |
| branch = st.secrets.get("HF_DEFAULT_BRANCH", DEFAULT_HF_BRANCH) | |
| subfolder = st.secrets.get("HF_DEFAULT_SUBFOLDER", DEFAULT_HF_SUBFOLDER) | |
| except Exception: | |
| token = os.environ.get("HF_TOKEN") or None | |
| branch = os.environ.get("HF_DEFAULT_BRANCH", DEFAULT_HF_BRANCH) | |
| subfolder = os.environ.get("HF_DEFAULT_SUBFOLDER", DEFAULT_HF_SUBFOLDER) | |
| with st.spinner("Loading evaluation data…"): | |
| local_root = dashboard._hf_snapshot(branch, token) | |
| candidate = os.path.join(local_root, subfolder) | |
| return candidate if os.path.isdir(candidate) else local_root | |
| def main(): | |
| st.set_page_config(page_title="SRMD Shabad Live Translation Evaluations", layout="wide", | |
| initial_sidebar_state="expanded") | |
| # SRMD logo beside the title, when the asset is available (in the Space build). | |
| logo = os.path.join(os.path.dirname(os.path.abspath(__file__)), "srmd_logo.jpg") | |
| if os.path.exists(logo): | |
| col_logo, col_title = st.columns([1, 9]) | |
| with col_logo: | |
| st.image(logo, width=72) | |
| with col_title: | |
| st.title("SRMD Shabad Live Translation Evaluations") | |
| else: | |
| st.title("SRMD Shabad Live Translation Evaluations") | |
| eval_dir = _resolve_eval_dir() | |
| raw_hdf = dashboard.load_holistic(eval_dir) | |
| raw_iedf = dashboard.load_intent_entity(eval_dir) | |
| if raw_hdf.empty and raw_iedf.empty: | |
| st.error(f"No results found in `{eval_dir}/`.") | |
| return | |
| discovered = sorted(set( | |
| list(raw_hdf["model"].unique() if not raw_hdf.empty else []) | |
| + list(raw_iedf["model"].unique() if not raw_iedf.empty else []) | |
| )) | |
| # Resolve our model folder from the candidates; fall back to the first | |
| # discovered folder so the dashboard still renders. | |
| ours = next((c for c in DEFAULT_OURS_CANDIDATES if c in discovered), discovered[0]) | |
| hdf = _relabel(raw_hdf, ours) | |
| iedf = _relabel(raw_iedf, ours) | |
| page = st.sidebar.radio( | |
| "Page", | |
| ["Model Quality Summary", "Issues Deep Dive"], | |
| ) | |
| if page == "Model Quality Summary": | |
| render_comparison(hdf, iedf) | |
| elif page == "Issues Deep Dive": | |
| render_issue_numbers(hdf) | |
| if __name__ == "__main__": | |
| main() | |