drop results path if queried word not even in inflections data

app.py

@@ -1837,20 +1837,32 @@ def _wiktionary_find_all_entries(word: str, conn: sqlite3.Connection) -> List[Di
 
 def _wiktionary_format_semantics_block(
     wikt_report: Dict[str, Any], 
+    pattern_block: Dict[str, Any], 
     top_n: int
 ) -> Dict[str, Any]:
     """
     Combines Wiktionary senses with OdeNet/ConceptNet senses,
-    using the ground-truth lemma from THIS Wiktionary report.
+    using the CORRECT lemma from the pattern.de analysis block.
     """
     
     # --- THIS IS THE FIX ---
-    # The semantic lemma IS the lemma from this specific wiktionary entry.
-    # e.g., for the "heuen" entry, this will be "heuen".
-    # e.g., for the "heute" entry, this will be "heute".
-    semantic_lemma = wikt_report.get("lemma", "")
+    # Determine the true lemma from the pattern.de block, as it's more reliable
+    # for semantic lookup than the wiktionary lemma (which could be an inflected form).
     pos_key = _wiktionary_map_pos_key(wikt_report.get("pos"))
-    log(f"Wiktionary Semantics: Building block for lemma='{semantic_lemma}', pos='{pos_key}'")
+    
+    semantic_lemma = ""
+    if pos_key == "verb":
+        semantic_lemma = pattern_block.get("infinitive")
+    elif pos_key == "noun":
+        semantic_lemma = pattern_block.get("base_form")
+    elif pos_key == "adjective":
+        semantic_lemma = pattern_block.get("predicative")
+    
+    # Fallback if pattern.de fails or it's a non-inflecting POS
+    if not semantic_lemma:
+         semantic_lemma = wikt_report.get("lemma", "") 
+    
+    log(f"[DEBUG] Wiktionary Semantics: Building block for lemma='{semantic_lemma}', pos='{pos_key}'")
     # --- END OF FIX ---
 
     # 1. Get Wiktionary senses (from the original report)
@@ -1872,7 +1884,7 @@ def _wiktionary_format_semantics_block(
             if odenet_senses_raw and "info" not in odenet_senses_raw[0]:
                 odenet_senses = odenet_senses_raw
         except Exception as e:
-            log(f"OdeNet lookup failed for {semantic_lemma} ({pos_key}): {e}")
+            log(f"[DEBUG] OdeNet lookup failed for {semantic_lemma} ({pos_key}): {e}")
 
     # 3. Get ConceptNet relations for the *semantic_lemma*
     conceptnet_relations = []
@@ -1922,17 +1934,24 @@ def _analyze_word_with_wiktionary(word: str, top_n: int) -> Dict[str, Any]:
             if iwnlp:
                 doc = iwnlp(word)
                 token = doc[0]
-                spacy_pos_hint = token.pos_.lower()
+                # Map spaCy POS to our internal keys
+                spacy_pos_raw = token.pos_.lower()
+                if spacy_pos_raw == "adj": spacy_pos_hint = "adjective"
+                elif spacy_pos_raw == "adv": spacy_pos_hint = "adverb"
+                elif spacy_pos_raw == "verb": spacy_pos_hint = "verb"
+                elif spacy_pos_raw == "noun": spacy_pos_hint = "noun"
+                else: spacy_pos_hint = spacy_pos_raw
+                    
                 spacy_lemma_hint = token.lemma_
-                log(f"Wiktionary Priority Hint: spaCy POS is '{spacy_pos_hint}', lemma is '{spacy_lemma_hint}'")
+                log(f"[DEBUG] Wiktionary Priority Hint: spaCy POS is '{spacy_pos_hint}', lemma is '{spacy_lemma_hint}'")
         except Exception as e:
-            log(f"Wiktionary Priority Hint: spaCy/IWNLP failed: {e}")
+            log(f"[DEBUG] Wiktionary Priority Hint: spaCy/IWNLP failed: {e}")
 
     # --- 2. FIND ALL WIKTIONARY ENTRIES ---
     try:
         wiktionary_reports = _wiktionary_find_all_entries(word, conn)
     except Exception as e:
-        log(f"Wiktionary query failed: {e}")
+        log(f"[DEBUG] Wiktionary query failed: {e}")
         return {} # Signal failure
 
     if not wiktionary_reports:
@@ -1945,11 +1964,11 @@ def _analyze_word_with_wiktionary(word: str, top_n: int) -> Dict[str, Any]:
         
         # Priority 1: Exact POS match with spaCy hint
         if spacy_pos_hint and wikt_pos == spacy_pos_hint:
-             # Bonus if lemma also matches
-             if spacy_lemma_hint and wikt_lemma == spacy_lemma_hint:
-                 return 1
-             return 2
-             
+            # Bonus if lemma also matches
+            if spacy_lemma_hint and wikt_lemma == spacy_lemma_hint:
+                return 1
+            return 2
+            
         # Priority 2: Input word is the lemma (e.g., "Haus" -> "Haus")
         if wikt_lemma.lower() == word.lower():
             return 3
@@ -1958,59 +1977,96 @@ def _analyze_word_with_wiktionary(word: str, top_n: int) -> Dict[str, Any]:
         return 4
 
     wiktionary_reports.sort(key=get_priority_score)
-    log(f"Wiktionary: Sorted entries: {[r.get('lemma') + ' (' + r.get('pos') + ')' for r in wiktionary_reports]}")
+    log(f"[DEBUG] Wiktionary: Sorted entries: {[r.get('lemma') + ' (' + r.get('pos') + ')' for r in wiktionary_reports]}")
 
 
-    # --- 4. BUILD THE FINAL REPORT (PATH-PURE) ---
+    # --- 4. BUILD AND *VALIDATE* THE FINAL REPORT (PATH-PURE) ---
+    word_lower = word.lower()
+    
     for wikt_report in wiktionary_reports:
         pos_key = _wiktionary_map_pos_key(wikt_report.get("pos"))
         lemma = wikt_report.get("lemma", word)
+        pos_title = wikt_report.get("pos_title", "")
         
-        # --- A. Build Semantics Block (USING WIKT LEMMA) ---
-        # This is the pure path. "heuen" entry looks up "heuen". "heute" entry looks up "heute".
-        semantics_block = _wiktionary_format_semantics_block(wikt_report, top_n)
-        
-        # --- B. Build Wiktionary Inflection Block ---
+        # --- A. Build Wiktionary Inflection Block ---
         inflections_wikt_block = {
             "base_form": lemma,
             "forms_list": wikt_report.get("forms", []),
             "source": "wiktionary"
         }
         
-        # --- C. Build Pattern Inflection Block (for comparison) ---
+        # --- B. Build Pattern Inflection Block (CRITICAL for finding true lemma) ---
         pattern_block = {}
         if PATTERN_DE_AVAILABLE:
             try:
-                if pos_key == "noun":
+                if pos_key == "noun" or "Substantiv" in pos_title:
                     pattern_block = pattern_analyze_as_noun(lemma)
-                elif pos_key == "verb":
-                    pattern_block = pattern_analyze_as_verb(lemma)
-                elif pos_key == "adjective":
-                    pattern_block = pattern_analyze_as_adjective(lemma)
+                elif pos_key == "verb" or "Verb" in pos_title or "Konjugierte Form" in pos_title:
+                    # Use the *input word* for inflected forms to find the right lemma
+                    if "Konjugierte Form" in pos_title:
+                        pattern_block = pattern_analyze_as_verb(word)
+                    else:
+                        pattern_block = pattern_analyze_as_verb(lemma)
+                elif pos_key == "adjective" or "Adjektiv" in pos_title or "Deklinierte Form" in pos_title:
+                    # Use the *input word* for inflected forms
+                    if "Deklinierte Form" in pos_title:
+                         pattern_block = pattern_analyze_as_adjective(word)
+                    else:
+                         pattern_block = pattern_analyze_as_adjective(lemma)
                 elif pos_key == "adverb":
                     pattern_block = {"base_form": lemma, "info": "Adverbs are non-inflecting."}
             except Exception as e:
                 pattern_block = {"error": f"Pattern.de analysis for {pos_key}('{lemma}') failed: {e}"}
 
-        # --- D. Assemble the final report for this entry ---
+        # --- C. Build Semantics Block (using correct lemma from pattern_block) ---
+        semantics_block = _wiktionary_format_semantics_block(wikt_report, pattern_block, top_n)
+        
+        # --- D. Assemble the report (pre-validation) ---
         pos_entry_report = {
             "inflections_wiktionary": inflections_wikt_block,
             "inflections_pattern": pattern_block,
             "semantics_combined": semantics_block,
             "wiktionary_metadata": {
-                "pos_title": wikt_report.get("pos_title"),
+                "pos_title": pos_title,
                 "pronunciation": wikt_report.get("sounds"),
                 "examples": wikt_report.get("examples")
             }
         }
         
-        # Append to the list for this POS key
-        if pos_key not in final_result["analysis"]:
-            final_result["analysis"][pos_key] = []
+        # --- E. *** YOUR NEW VALIDATION FILTER (Corrected) *** ---
+        is_valid = False
+        is_inflected_entry = "Konjugierte Form" in pos_title or "Deklinierte Form" in pos_title
+
+        # Check 1: Is the input word the lemma OF A BASE FORM entry?
+        if not is_inflected_entry and lemma.lower() == word_lower:
+            is_valid = True
+            log(f"[DEBUG] Wiktionary: KEEPING entry '{lemma}' ({pos_key}) because input word matches lemma of a base entry.")
         
-        final_result["analysis"][pos_key].append(pos_entry_report)
+        # Check 2: Is the input word in the *bare* forms list?
+        # (This is the only check that should apply to inflected entries)
+        if not is_valid:
+            for form_entry in inflections_wikt_block.get("forms_list", []):
+                form_text = form_entry.get("form_text", "")
+                bare_form = re.sub(r"\(.*\)", "", form_text).strip()
+                bare_form = re.sub(r"^(der|die|das|ein|eine|am)\s+", "", bare_form, flags=re.IGNORECASE).strip()
+                bare_form = bare_form.rstrip("!.")
+                
+                if bare_form.lower() == word_lower:
+                    is_valid = True
+                    log(f"[DEBUG] Wiktionary: KEEPING entry '{lemma}' ({pos_key}) because input word found in form: '{form_text}'")
+                    break
+        
+        # --- F. Add to final result if valid ---
+        if is_valid:
+            if pos_key not in final_result["analysis"]:
+                final_result["analysis"][pos_key] = []
+            final_result["analysis"][pos_key].append(pos_entry_report)
+        else:
+             log(f"[DEBUG] Wiktionary: DROPPING entry '{lemma}' ({pos_key}, {pos_title}) because input word '{word}' was not found in its valid forms.")
+    
+    # --- END OF VALIDATION ---
 
-    final_result["info"] = f"Analysis from Wiktionary (Primary Engine). Found {len(wiktionary_reports)} matching entry/entries."
+    final_result["info"] = f"Analysis from Wiktionary (Primary Engine). Found {len(wiktionary_reports)} matching entries, kept {sum(len(v) for v in final_result.get('analysis', {}).values())}."
     return final_result
 
 # ============================================================================
@@ -2708,58 +2764,57 @@ def _analyze_word_with_iwnlp(word: str, top_n_value: int) -> Dict[str, Any]:
 # --- 7b. NEW: Word Encyclopedia (Non-Contextual) Analyzer ---
 
 # --- THIS IS THE NEW PUBLIC DISPATCHER FUNCTION ---
-def analyze_word_encyclopedia(word: str, top_n_value: Optional[float] = 0) -> Dict[str, Any]:
+def analyze_word_encyclopedia(word: str, top_n_value: Optional[float] = 0, engine_choice: str = "wiktionary") -> Dict[str, Any]:
     """
-    (PUBLIC DISPATCHER V19) Analyzes a single word for all possible forms.
+    (PUBLIC DISPATCHER V20) Analyzes a single word using the selected engine.
     
-    This function intelligently selects the best available engine in order:
-    1.  PRIMARY:   Wiktionary DB (Accurate, pre-compiled data)
-    2.  FALLBACK 1: HanTa-led engine (Good heuristics)
-    3.  FALLBACK 2: IWNLP-led engine (Different heuristics)
+    The user can now choose which engine to run.
     """
     if not word or not word.strip():
         return {"info": "Please enter a word."}
     
     word = word.strip()
     top_n = int(top_n_value) if top_n_value is not None else 0
+    result = {}
 
-    # --- 1. Try PRIMARY Engine: Wiktionary ---
-    if WIKTIONARY_AVAILABLE:
-        wikt_result = _analyze_word_with_wiktionary(word, top_n)
-        if wikt_result and wikt_result.get("analysis"):
-            log("V19 Dispatcher: Returning Wiktionary result.")
-            return wikt_result
-        elif WIKTIONARY_AVAILABLE:
-             log("V19 Dispatcher: Wiktionary is available but found no results.")
-        else:
-             log("V19 Dispatcher: Wiktionary failed to initialize, falling back.")
+    log(f"\n[Word Encyclopedia] User selected engine: '{engine_choice}' for word: '{word}'")
 
-    # --- 2. Try FALLBACK 1: HanTa ---
-    if HANTA_AVAILABLE:
-        hanta_result = _analyze_word_with_hanta(word, top_n)
-        if hanta_result and hanta_result.get("analysis"):
-            log("V19 Dispatcher: Wiktionary failed, returning HanTa result.")
-            return hanta_result
+    try:
+        if engine_choice == "wiktionary":
+            result = _analyze_word_with_wiktionary(word, top_n)
+            if not result or not result.get("analysis"):
+                 result["info"] = f"Wiktionary (Primary Engine) found no results for '{word}'. You can try a fallback engine."
+
+        elif engine_choice == "hanta":
+            result = _analyze_word_with_hanta(word, top_n)
+            if not result or not result.get("analysis"):
+                 result["info"] = f"HanTa (Fallback 1) found no results for '{word}'."
+
+        elif engine_choice == "iwnlp":
+            result = _analyze_word_with_iwnlp(word, top_n)
+            if not result or not result.get("analysis"):
+                 result["info"] = f"IWNLP (Fallback 2) found no results for '{word}'."
+        
         else:
-            log("V19 Dispatcher: HanTa fallback found no results.")
+            result = {"error": f"Unknown engine choice: {engine_choice}"}
             
-    # --- 3. Try FALLBACK 2: IWNLP ---
-    if IWNLP_AVAILABLE:
-        iwnlp_result = _analyze_word_with_iwnlp(word, top_n)
-        if iwnlp_result and iwnlp_result.get("analysis"):
-            log("V19 Dispatcher: HanTa failed, returning IWNLP result.")
-            return iwnlp_result
-        else:
-            log("V19 Dispatcher: IWNLP fallback found no results.")
+    except Exception as e:
+        log(f"--- Dispatcher FAILED for engine {engine_choice}: {e} ---")
+        traceback.print_exc()
+        return {
+            "input_word": word,
+            "error": f"The '{engine_choice}' engine failed during analysis.",
+            "traceback": traceback.format_exc()
+        }
 
-    # --- No engines available or no results ---
-    log("--- Dispatcher: No valid analysis engines found or no results. ---")
-    return {
-        "input_word": word,
-        "error": "No analysis found for this word.",
-        "info": "The word was not found in Wiktionary, and no fallback "
-                "engines (HanTa, IWNLP) could produce a valid analysis."
-    }
+    # If the engine ran but found nothing, return a clear info message
+    if not result.get("analysis"):
+         return {
+             "input_word": word,
+             "info": result.get("info", f"The selected engine '{engine_choice}' found no valid analysis for this word.")
+         }
+
+    return result
 
 
 # ============================================================================
@@ -2955,40 +3010,136 @@ def create_combined_tab():
 def create_word_encyclopedia_tab():
     """--- NEW: Creates the UI for the NON-CONTEXTUAL Word Analyzer tab ---"""
     gr.Markdown("# 📖 Word Encyclopedia (Non-Contextual)")
-    gr.Markdown("This tool analyzes a **single word** for *all possible* grammatical and semantic forms. It's ideal for enriching word lists. It finds ambiguities (e.g., 'Lauf' as noun and verb) and groups all data by Part-of-Speech.")
+    gr.Markdown("This tool analyzes a **single word** for *all possible* grammatical and semantic forms. It finds ambiguities (e.g., 'Lauf' as noun and verb) and groups all data by Part-of-Speech.")
+    
     with gr.Column():
         word_input = gr.Textbox(
             label="Single German Word",
-            placeholder="e.g., Lauf, See, schnell"
-        )
-        top_n_number = gr.Number(
-            label="Limit Semantic Senses per POS (0 for all)",
-            value=0,
-            step=1,
-            minimum=0,
-            interactive=True
+            placeholder="e.g., Lauf, See, schnell, heute"
         )
+        
+        with gr.Row():
+            top_n_number = gr.Number(
+                label="Limit Semantic Senses per POS (0 for all)",
+                value=0,
+                step=1,
+                minimum=0,
+                interactive=True
+            )
+            
+            # --- THIS IS THE NEW UI ELEMENT ---
+            engine_radio = gr.Radio(
+                label="Select Analysis Engine",
+                choices=[
+                    ("Wiktionary (Default)", "wiktionary"),
+                    ("HanTa (Fallback 1)", "hanta"),
+                    ("IWNLP (Fallback 2)", "iwnlp")
+                ],
+                value="wiktionary",
+                interactive=True
+            )
+            # --- END OF NEW UI ELEMENT ---
+
         analyze_button = gr.Button("Analyze Word", variant="primary")
     
     output = gr.JSON(label="Word Encyclopedia Analysis (JSON)")
     
+    # --- UPDATE THE CLICK FUNCTION ---
     analyze_button.click(
         fn=analyze_word_encyclopedia,
-        inputs=[word_input, top_n_number],
+        # Add 'engine_radio' to the inputs
+        inputs=[word_input, top_n_number, engine_radio], 
         outputs=[output],
         api_name="analyze_word"
     )
     
+    # Update the examples to include the radio button
     gr.Examples(
-        [["Lauf", 3], 
-         ["See", 0], 
-         ["schnell", 3],
-         ["Hund", 5]],
-        inputs=[word_input, top_n_number],
+        [["Lauf", 3, "wiktionary"], 
+         ["See", 0, "wiktionary"], 
+         ["schnell", 3, "wiktionary"],
+         ["heute", 0, "wiktionary"],
+         ["heute", 0, "hanta"]], # Example to show a different engine
+        inputs=[word_input, top_n_number, engine_radio],
         outputs=[output], 
         fn=analyze_word_encyclopedia
     )
 
+def create_wiktionary_tab():
+    """Creates the UI for the standalone Wiktionary lookup tab."""
+    gr.Markdown("# 📙 Wiktionary Lookup (Raw Engine)")
+    gr.Markdown("Directly query the Wiktionary (Primary) engine. This shows the raw, combined data from the database, Pattern.de, and semantic sources.")
+    with gr.Column():
+        word_input = gr.Textbox(
+            label="Single German Word",
+            placeholder="e.g., Haus, gehe, heute"
+        )
+        analyze_button = gr.Button("Lookup Word in Wiktionary", variant="primary")
+    
+    output = gr.JSON(label="Wiktionary Engine Analysis (JSON)")
+    
+    # Call the internal engine function directly, hardcoding top_n=0
+    analyze_button.click(
+        fn=lambda word: _analyze_word_with_wiktionary(word, 0),
+        inputs=[word_input],
+        outputs=[output],
+        api_name="wiktionary_lookup"
+    )
+    gr.Examples(
+        [["Haus"], ["gehe"], ["heute"], ["Lauf"]],
+        inputs=[word_input], outputs=[output], fn=lambda word: _analyze_word_with_wiktionary(word, 0)
+    )
+
+def create_hanta_tab():
+    """Creates the UI for the standalone HanTa Engine tab."""
+    gr.Markdown("# 🤖 HanTa Lookup (Raw Engine)")
+    gr.Markdown("Directly query the HanTa (Fallback 1) engine. This shows the raw, combined data from HanTa, Pattern.de, and semantic sources.")
+    with gr.Column():
+        word_input = gr.Textbox(
+            label="Single German Word",
+            placeholder="e.g., Haus, gehe, heute"
+        )
+        analyze_button = gr.Button("Lookup Word with HanTa", variant="primary")
+    
+    output = gr.JSON(label="HanTa Engine Analysis (JSON)")
+    
+    # Call the internal engine function directly, hardcoding top_n=0
+    analyze_button.click(
+        fn=lambda word: _analyze_word_with_hanta(word, 0),
+        inputs=[word_input],
+        outputs=[output],
+        api_name="hanta_lookup"
+    )
+    gr.Examples(
+        [["Haus"], ["gehe"], ["heute"], ["Lauf"]],
+        inputs=[word_input], outputs=[output], fn=lambda word: _analyze_word_with_hanta(word, 0)
+    )
+
+def create_iwnlp_tab():
+    """Creates the UI for the standalone IWNLP Engine tab."""
+    gr.Markdown("# 🔬 IWNLP-spaCy Lookup (Raw Engine)")
+    gr.Markdown("Directly query the IWNLP-spaCy (Fallback 2) engine. This shows the raw, combined data from spaCy, IWNLP, Pattern.de, and semantic sources.")
+    with gr.Column():
+        word_input = gr.Textbox(
+            label="Single German Word",
+            placeholder="e.g., Haus, gehe, heute"
+        )
+        analyze_button = gr.Button("Lookup Word with IWNLP", variant="primary")
+    
+    output = gr.JSON(label="IWNLP Engine Analysis (JSON)")
+    
+    # Call the internal engine function directly, hardcoding top_n=0
+    analyze_button.click(
+        fn=lambda word: _analyze_word_with_iwnlp(word, 0),
+        inputs=[word_input],
+        outputs=[output],
+        api_name="iwnlp_lookup"
+    )
+    gr.Examples(
+        [["Haus"], ["gehe"], ["heute"], ["Lauf"]],
+        inputs=[word_input], outputs=[output], fn=lambda word: _analyze_word_with_iwnlp(word, 0)
+    )
+
 # --- Main UI Builder ---
 def create_consolidated_interface():
     """Builds the final Gradio app with all tabs."""
@@ -2997,7 +3148,7 @@ def create_consolidated_interface():
         gr.Markdown("A suite of advanced tools for German linguistics, providing both contextual and non-contextual analysis.")
         
         with gr.Tabs():
-            # --- NEW "Word Encyclopedia" TAB ---
+            # --- Main Tools ---
             with gr.Tab("📖 Word Encyclopedia (DE)"):
                 create_word_encyclopedia_tab()
                 
@@ -3009,14 +3160,25 @@ def create_consolidated_interface():
             
             with gr.Tab("✅ Grammar Check (DE)"):
                 create_languagetool_tab()
+            
+            # --- Standalone Engine Tabs (NEW) ---
+            with gr.Tab("📙 Engine: Wiktionary (DE)"):
+                create_wiktionary_tab()
 
-            with gr.Tab("📚 Inflections (DE)"):
+            with gr.Tab("🤖 Engine: HanTa (DE)"):
+                create_hanta_tab()
+
+            with gr.Tab("🔬 Engine: IWNLP-spaCy (DE)"):
+                create_iwnlp_tab()
+                
+            # --- Standalone Component Tabs ---
+            with gr.Tab("📚 Component: Inflections (DE)"):
                 create_pattern_tab()
 
-            with gr.Tab("📖 Thesaurus (DE)"):
+            with gr.Tab("📖 Component: Thesaurus (DE)"):
                 create_odenet_tab()
             
-            with gr.Tab("🌐 ConceptNet (Direct)"):
+            with gr.Tab("🌐 Component: ConceptNet (Direct)"):
                 create_conceptnet_tab()
     
     return demo