Upload folder using huggingface_hub

app.py

@@ -3,7 +3,8 @@ Gradio Web Application for Neural Transducer Morphological Inflection.
 Features:
 - Language selection (English, German, French, Italian)
 - Side-by-side comparison (100 examples vs 1,000 examples)
-- Decoding mode (Greedy vs Beam search)
+- Random sampling from official CoNLL-SIGMORPHON 2017 Dev and Test splits with gold target display
+- Training exposure indicators (Lemma seen? Feature combination seen?) for both regimes
 - Detailed action alignment traces (COPY, DELETE, INSERT, SUBSTITUTE)
 - Curated linguistic examples
 """
@@ -40,7 +41,26 @@ def format_action_trace(trace):
     return "\n".join(rows)
 
 
-def inflect(language_label, mode, lemma, features_str):
+def build_coverage_markdown(lang_iso, lemma, features_str):
+    if not lemma or not features_str:
+        return ""
+    cov = engine.check_training_coverage(lang_iso, lemma, features_str)
+    low_lemma = "✅ Seen in training" if cov["100"]["lemma_seen"] else "❌ Unseen (zero-shot lemma)"
+    med_lemma = "✅ Seen in training" if cov["1000"]["lemma_seen"] else "❌ Unseen (zero-shot lemma)"
+
+    low_feat = "✅ Seen in training" if cov["100"]["features_seen"] else "❌ Unseen combination"
+    med_feat = "✅ Seen in training" if cov["1000"]["features_seen"] else "❌ Unseen combination"
+
+    md = f"""
+| Training Exposure | 100 Examples (Low) | 1,000 Examples (Medium) |
+| :--- | :--- | :--- |
+| **Lemma** (`{lemma}`) | {low_lemma} | {med_lemma} |
+| **Features** (`{features_str}`) | {low_feat} | {med_feat} |
+"""
+    return md.strip()
+
+
+def inflect(language_label, mode, lemma, features_str, gold_target=""):
     if not lemma or not features_str:
         return (
             "Please provide both a lemma and UniMorph features.",
@@ -48,11 +68,15 @@ def inflect(language_label, mode, lemma, features_str):
             "-",
             "Waiting for valid input...",
             "",
+            "",
         )
 
     iso = LANGUAGE_MAP.get(language_label, "eng")
     lemma = lemma.strip()
     features_str = features_str.strip()
+    gold_target = (gold_target or "").strip()
+
+    cov_md = build_coverage_markdown(iso, lemma, features_str)
 
     try:
         if mode == "Compare Both (100 vs 1000)":
@@ -60,9 +84,15 @@ def inflect(language_label, mode, lemma, features_str):
             res_1000 = engine.predict(iso, "1000", lemma, features_str, beam_width=1)
 
             pred_100 = res_100.predicted_form
+            if gold_target:
+                status_100 = " ✅ Match" if pred_100 == gold_target else " ❌ Mismatch"
+                pred_100 = f"{pred_100}{status_100}"
             score_100 = f"{res_100.decoder_score:.4f}"
 
             pred_1000 = res_1000.predicted_form
+            if gold_target:
+                status_1000 = " ✅ Match" if pred_1000 == gold_target else " ❌ Mismatch"
+                pred_1000 = f"{pred_1000}{status_1000}"
             score_1000 = f"{res_1000.decoder_score:.4f}"
 
             trace_text = (
@@ -72,22 +102,36 @@ def inflect(language_label, mode, lemma, features_str):
 
             warn = ""
             if res_1000.unknown_features:
-                warn = f"⚠️ Unseen feature tags (treated as UNK): {', '.join(res_1000.unknown_features)}"
+                warn = f"⚠️ Unseen individual tag tokens (treated as UNK): {', '.join(res_1000.unknown_features)}"
 
-            return pred_100, score_100, pred_1000, score_1000, trace_text, warn
+            return pred_100, score_100, pred_1000, score_1000, trace_text, warn, cov_md
 
         elif mode == "100 Examples (Low-Resource)":
             res = engine.predict(iso, "100", lemma, features_str, beam_width=1)
-            warn = f"⚠️ Unseen feature tags: {', '.join(res.unknown_features)}" if res.unknown_features else ""
-            return res.predicted_form, f"{res.decoder_score:.4f}", "-", "-", format_action_trace(res.action_trace), warn
+            pred = res.predicted_form
+            if gold_target:
+                pred += " ✅ Match" if pred == gold_target else " ❌ Mismatch"
+            warn = f"⚠️ Unseen individual tag tokens: {', '.join(res.unknown_features)}" if res.unknown_features else ""
+            return pred, f"{res.decoder_score:.4f}", "-", "-", format_action_trace(res.action_trace), warn, cov_md
 
         else:  # 1000 Examples
             res = engine.predict(iso, "1000", lemma, features_str, beam_width=1)
-            warn = f"⚠️ Unseen feature tags: {', '.join(res.unknown_features)}" if res.unknown_features else ""
-            return "-", "-", res.predicted_form, f"{res.decoder_score:.4f}", format_action_trace(res.action_trace), warn
+            pred = res.predicted_form
+            if gold_target:
+                pred += " ✅ Match" if pred == gold_target else " ❌ Mismatch"
+            warn = f"⚠️ Unseen individual tag tokens: {', '.join(res.unknown_features)}" if res.unknown_features else ""
+            return "-", "-", pred, f"{res.decoder_score:.4f}", format_action_trace(res.action_trace), warn, cov_md
 
     except Exception as e:
-        return f"Error: {e}", "-", "-", "-", str(e), ""
+        return f"Error: {e}", "-", "-", "-", str(e), "", cov_md
+
+
+def sample_random_item(language_label, split_choice):
+    iso = LANGUAGE_MAP.get(language_label, "eng")
+    sample = engine.get_random_sample(iso, split=split_choice)
+    if not sample:
+        return "sing", "V;PST", "", "No samples available."
+    return sample["lemma"], sample["features"], sample["target"]
 
 
 # Gradio UI definition
@@ -112,6 +156,17 @@ with gr.Blocks(title="Neural Transducer: Morphological Inflection", theme=gr.the
                 choices=["Compare Both (100 vs 1000)", "100 Examples (Low-Resource)", "1000 Examples (Medium-Resource)"],
                 value="Compare Both (100 vs 1000)",
             )
+
+            with gr.Group():
+                gr.Markdown("#### 🎲 Benchmark Evaluation Sampler")
+                with gr.Row():
+                    split_selector = gr.Radio(
+                        label="Benchmark Split",
+                        choices=["Dev Set", "Test Set"],
+                        value="Dev Set",
+                    )
+                    sample_btn = gr.Button("🎲 Sample Random Item", variant="secondary")
+
             lemma_input = gr.Textbox(
                 label="Lemma",
                 placeholder="e.g. sing",
@@ -122,6 +177,12 @@ with gr.Blocks(title="Neural Transducer: Morphological Inflection", theme=gr.the
                 placeholder="e.g. V;PST",
                 value="V;PST",
             )
+            gold_target_input = gr.Textbox(
+                label="Gold Target Form (Optional / Benchmark Reference)",
+                placeholder="Populated automatically when sampling from Dev/Test",
+                value="",
+            )
+
             run_btn = gr.Button("Generate Inflection", variant="primary")
 
         with gr.Column(scale=1):
@@ -136,26 +197,40 @@ with gr.Blocks(title="Neural Transducer: Morphological Inflection", theme=gr.the
 
                 warning_box = gr.Markdown("")
 
+            with gr.Group():
+                gr.Markdown("### 📊 Training Data Exposure")
+                coverage_box = gr.Markdown("Enter an input or sample from the benchmark to view coverage.")
+
             with gr.Accordion("🔍 Transducer Edit Action Trace (Alignment Steps)", open=False):
                 trace_box = gr.Code(label="Action Sequence", language="markdown", interactive=False)
 
     gr.Markdown("### 💡 Click an Example to Load")
     example_rows = []
     for ex in CURATED_EXAMPLES:
-        example_rows.append([ex["language"], "Compare Both (100 vs 1000)", ex["lemma"], ex["features"]])
+        example_rows.append([ex["language"], "Compare Both (100 vs 1000)", ex["lemma"], ex["features"], ""])
 
     gr.Examples(
         examples=example_rows,
-        inputs=[lang_dropdown, mode_radio, lemma_input, feat_input],
-        outputs=[out_100, score_100, out_1000, score_1000, trace_box, warning_box],
+        inputs=[lang_dropdown, mode_radio, lemma_input, feat_input, gold_target_input],
+        outputs=[out_100, score_100, out_1000, score_1000, trace_box, warning_box, coverage_box],
         fn=inflect,
         cache_examples=False,
     )
 
+    sample_btn.click(
+        fn=sample_random_item,
+        inputs=[lang_dropdown, split_selector],
+        outputs=[lemma_input, feat_input, gold_target_input],
+    ).then(
+        fn=inflect,
+        inputs=[lang_dropdown, mode_radio, lemma_input, feat_input, gold_target_input],
+        outputs=[out_100, score_100, out_1000, score_1000, trace_box, warning_box, coverage_box],
+    )
+
     run_btn.click(
         fn=inflect,
-        inputs=[lang_dropdown, mode_radio, lemma_input, feat_input],
-        outputs=[out_100, score_100, out_1000, score_1000, trace_box, warning_box],
+        inputs=[lang_dropdown, mode_radio, lemma_input, feat_input, gold_target_input],
+        outputs=[out_100, score_100, out_1000, score_1000, trace_box, warning_box, coverage_box],
     )
 
 if __name__ == "__main__":

inference.py

@@ -83,3 +83,49 @@ class TransducerSpaceEngine:
             features_str=features,
             beam_width=beam_width,
         )
+
+    def _load_data_index(self) -> Dict:
+        if not hasattr(self, "_data_index"):
+            index_path = os.path.join(os.path.dirname(__file__), "data_index.json")
+            if not os.path.exists(index_path):
+                # Fallback to demo_core location if needed
+                index_path = os.path.join(os.path.dirname(__file__), "..", "demo_core", "data_index.json")
+            if os.path.exists(index_path):
+                with open(index_path, "r", encoding="utf-8") as f:
+                    self._data_index = json.load(f)
+            else:
+                self._data_index = {}
+        return self._data_index
+
+    def get_random_sample(self, lang: str, split: str = "dev") -> Optional[Dict[str, str]]:
+        import random
+        data = self._load_data_index()
+        split_key = "dev" if split.lower().startswith("dev") else "test"
+        samples = data.get(lang, {}).get(split_key, [])
+        if not samples:
+            return None
+        return random.choice(samples)
+
+    def check_training_coverage(self, lang: str, lemma: str, features_str: str) -> Dict[str, Dict[str, bool]]:
+        data = self._load_data_index()
+        lang_data = data.get(lang, {}).get("training_coverage", {})
+        lemma = lemma.strip()
+        features_norm = set(features_str.strip().split(";"))
+
+        result = {}
+        for regime in ["100", "1000"]:
+            cov = lang_data.get(regime, {})
+            seen_lemmas = set(cov.get("lemmas", []))
+            raw_feats = cov.get("features", [])
+            seen_feats_sets = [set(f.split(";")) for f in raw_feats]
+
+            lemma_seen = lemma in seen_lemmas
+            # Feat combination seen if exact string matches or tag set matches
+            feats_seen = (features_str.strip() in raw_feats) or (features_norm in seen_feats_sets)
+
+            result[regime] = {
+                "lemma_seen": lemma_seen,
+                "features_seen": feats_seen,
+            }
+        return result
+