wiktionary diagnostics, fix NoneType

app.py

@@ -1270,6 +1270,55 @@ def wiktionary_download_db() -> bool:
         # traceback.print_exc() # Uncomment for deep debugging
         return False
 
+def wiktionary_run_startup_diagnostics():
+    """ Runs critical checks on the DB structure and content at startup. """
+    print("\n" + "="*50)
+    print("RUNNING WIKTIONARY DB DIAGNOSTICS")
+    print("="*50)
+    
+    conn = wiktionary_get_connection()
+    if not conn:
+        print("✗ Diagnostics aborted: No DB connection.")
+        return
+
+    try:
+        # 1. Check Table Structure
+        print("[1] Checking Tables...")
+        tables = conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()
+        table_names = [t['name'] for t in tables]
+        print(f"    Found tables: {table_names}")
+        
+        if 'entries' not in table_names:
+            print("CRITICAL ERROR: 'entries' table missing!")
+            return
+
+        # 2. Check Language Encoding (The likely cause of your empty results)
+        print("\n[2] Checking Language Format (Top 5)...")
+        langs = conn.execute("SELECT lang, COUNT(*) as c FROM entries GROUP BY lang ORDER BY c DESC LIMIT 5").fetchall()
+        for row in langs:
+            print(f"    - '{row['lang']}': {row['c']} entries")
+
+        # 3. Check Specific 'Missing' Words
+        test_words = ["ready", "runner", "run", "house"]
+        print(f"\n[3] Probing missing words: {test_words}")
+        for word in test_words:
+            # Check exact match raw
+            raw = conn.execute("SELECT count(*) as c FROM entries WHERE word = ?", (word,)).fetchone()
+            print(f"    - '{word}' (Raw check): Found {raw['c']} rows")
+            
+            if raw['c'] == 0:
+                # Check case insensitive
+                nocase = conn.execute("SELECT word FROM entries WHERE word LIKE ? LIMIT 1", (word,)).fetchone()
+                if nocase:
+                    print(f"      ! WARNING: '{word}' not found exactly, but found '{nocase['word']}' (Case mismatch?)")
+                else:
+                    print(f"      ! CRITICAL: '{word}' does not exist in DB at all.")
+
+    except Exception as e:
+        print(f"✗ Diagnostics crashed: {e}")
+        traceback.print_exc()
+    print("="*50 + "\n")
+
 def wiktionary_get_connection() -> Optional[sqlite3.Connection]:
     """ Thread-safe function to get a single, read-only SQLite connection. """
     global WIKTIONARY_CONN, WIKTIONARY_AVAILABLE
@@ -1568,15 +1617,13 @@ def _analyze_word_with_wiktionary(word: str, top_n: int) -> Dict[str, Any]:
         
     spacy_pos_hint, spacy_lemma_hint = None, None
     try:
-        # Quick heuristic to help sort results if multiple entries exist
         nlp_en = SPACY_MODELS.get("en")
         if nlp_en:
             doc = nlp_en(word)
             token = doc[0]
             spacy_pos_hint = token.pos_.lower()
             spacy_lemma_hint = token.lemma_
-    except Exception as e:
-        log(f"[DEBUG] Wiktionary (EN) Hint: spaCy failed: {e}")
+    except Exception: pass
 
     try:
         wiktionary_reports = _wiktionary_find_all_entries(word, conn)
@@ -1598,11 +1645,11 @@ def _analyze_word_with_wiktionary(word: str, top_n: int) -> Dict[str, Any]:
     
     word_lower = word.lower()
     for wikt_report in wiktionary_reports:
-        # --- FIX: Safe Extraction of DB Fields ---
+        # --- FIX START: Safe Extraction ---
         pos_key = _wiktionary_map_pos_key(wikt_report.get("pos"))
-        lemma = wikt_report.get("lemma") or word # Fallback if None
-        # FORCE STRING: Use 'or ""' to handle DB NULLs preventing "NoneType is not iterable"
-        pos_title = wikt_report.get("pos_title") or "" 
+        lemma = wikt_report.get("lemma") or word
+        pos_title = wikt_report.get("pos_title") or "" # FORCE STRING
+        # --- FIX END ---
         
         inflections_wikt_block = {
             "base_form": lemma,
@@ -1613,9 +1660,7 @@ def _analyze_word_with_wiktionary(word: str, top_n: int) -> Dict[str, Any]:
         pattern_block = {}
         if PATTERN_EN_AVAILABLE:
             try:
-                # Safe check now that pos_title is definitely a string
                 use_word = word if "form" in pos_title.lower() else lemma
-                
                 if pos_key == "noun": pattern_block = pattern_analyze_as_noun_en(use_word)
                 elif pos_key == "verb": pattern_block = pattern_analyze_as_verb_en(use_word)
                 elif pos_key == "adjective": pattern_block = pattern_analyze_as_adjective_en(use_word)
@@ -1636,9 +1681,7 @@ def _analyze_word_with_wiktionary(word: str, top_n: int) -> Dict[str, Any]:
             }
         }
         
-        # Validation Logic
         is_valid = False
-        # Safe check now that pos_title is definitely a string
         is_inflected_entry = any(ft in pos_title for ft in ["form", "Comparative", "Superlative"])
 
         if lemma.lower() == word_lower: is_valid = True
@@ -1654,10 +1697,8 @@ def _analyze_word_with_wiktionary(word: str, top_n: int) -> Dict[str, Any]:
             if pos_key not in final_result["analysis"]:
                 final_result["analysis"][pos_key] = []
             final_result["analysis"][pos_key].append(pos_entry_report)
-        else:
-             log(f"[DEBUG] Wiktionary (EN): DROPPING entry '{lemma}' ({pos_key}) ...")
     
-    final_result["info"] = f"Analysis from Wiktionary (Primary Engine). Found {len(wiktionary_reports)} matching entries."
+    final_result["info"] = f"Analysis from Wiktionary. Found {len(wiktionary_reports)} raw entries."
     return final_result
 
 
@@ -2250,11 +2291,16 @@ def _format_word_analysis_html(data: Dict[str, Any]) -> str:
         
         # --- Header ---
         pos_class = f"pos-{pos_key}" if pos_key in ["noun", "verb", "adj", "adv"] else "pos-noun"
-        # Try to find a lemma from one of the sub-blocks
-        lemma = entry.get("inflections_wiktionary", {}).get("base_form") or \
-                entry.get("inflections_pattern", {}).get("base_form") or \
-                entry.get("semantics_combined", {}).get("lemma") or \
-                data.get("input_word")
+        
+        # Safe extraction of lemma
+        inf_wikt = entry.get("inflections_wiktionary") or {}
+        inf_pat = entry.get("inflections_pattern") or {}
+        sem_comb = entry.get("semantics_combined") or {}
+        
+        lemma = inf_wikt.get("base_form") or \
+                inf_pat.get("base_form") or \
+                sem_comb.get("lemma") or \
+                data.get("input_word") or "?"
 
         html += f"""
         <div class="ling-card">
@@ -2268,70 +2314,73 @@ def _format_word_analysis_html(data: Dict[str, Any]) -> str:
         html += "<div class='ling-section'><div class='ling-subtitle'>Morphology & Inflections</div>"
         html += "<table class='inflection-table'>"
         
-        pat = entry.get("inflections_pattern", {})
-        wikt = entry.get("inflections_wiktionary", {})
-        
         # Noun Logic
         if pos_key == 'noun':
-            if pat:
-                html += f"<tr><td class='inflection-label'>Singular</td><td>{pat.get('singular', '-')}</td></tr>"
-                html += f"<tr><td class='inflection-label'>Plural</td><td>{pat.get('plural', '-')}</td></tr>"
-                html += f"<tr><td class='inflection-label'>Context</td><td>{pat.get('article', '-')}</td></tr>"
+            if inf_pat:
+                html += f"<tr><td class='inflection-label'>Singular</td><td>{inf_pat.get('singular', '-')}</td></tr>"
+                html += f"<tr><td class='inflection-label'>Plural</td><td>{inf_pat.get('plural', '-')}</td></tr>"
+                html += f"<tr><td class='inflection-label'>Context</td><td>{inf_pat.get('article', '-')}</td></tr>"
         
         # Verb Logic
         elif pos_key == 'verb':
-            cj = pat.get('conjugation', {})
-            if cj:
-                pres = cj.get('Present', {})
-                past = cj.get('Past', {})
-                html += f"<tr><td class='inflection-label'>Infinitive</td><td>{pat.get('infinitive', lemma)}</td></tr>"
-                html += f"<tr><td class='inflection-label'>3rd Person (He/She)</td><td>{pres.get('he/she (3sg)', '-')}</td></tr>"
-                html += f"<tr><td class='inflection-label'>Past Simple</td><td>{past.get('General', '-')}</td></tr>"
-                parts = pat.get('participles', {})
-                html += f"<tr><td class='inflection-label'>Participle (Ing)</td><td>{parts.get('Present Participle (gerund)', '-')}</td></tr>"
-                html += f"<tr><td class='inflection-label'>Participle (Past)</td><td>{parts.get('Past Participle', '-')}</td></tr>"
+            cj = inf_pat.get('conjugation') or {}
+            pres = cj.get('Present') or {}
+            past = cj.get('Past') or {}
+            parts = inf_pat.get('participles') or {}
+            
+            html += f"<tr><td class='inflection-label'>Infinitive</td><td>{inf_pat.get('infinitive', lemma)}</td></tr>"
+            html += f"<tr><td class='inflection-label'>3rd Person (He/She)</td><td>{pres.get('he/she (3sg)', '-')}</td></tr>"
+            html += f"<tr><td class='inflection-label'>Past Simple</td><td>{past.get('General', '-')}</td></tr>"
+            html += f"<tr><td class='inflection-label'>Participle (Ing)</td><td>{parts.get('Present Participle (gerund)', '-')}</td></tr>"
+            html += f"<tr><td class='inflection-label'>Participle (Past)</td><td>{parts.get('Past Participle', '-')}</td></tr>"
 
         # Adjective Logic
         elif pos_key == 'adjective':
-            gr = pat.get('grading', {})
-            if gr:
-                 html += f"<tr><td class='inflection-label'>Positive</td><td>{gr.get('Positive', '-')}</td></tr>"
-                 html += f"<tr><td class='inflection-label'>Comparative</td><td>{gr.get('Comparative', '-')}</td></tr>"
-                 html += f"<tr><td class='inflection-label'>Superlative</td><td>{gr.get('Superlative', '-')}</td></tr>"
+            gr = inf_pat.get('grading') or {}
+            html += f"<tr><td class='inflection-label'>Positive</td><td>{gr.get('Positive', '-')}</td></tr>"
+            html += f"<tr><td class='inflection-label'>Comparative</td><td>{gr.get('Comparative', '-')}</td></tr>"
+            html += f"<tr><td class='inflection-label'>Superlative</td><td>{gr.get('Superlative', '-')}</td></tr>"
 
         # Wiktionary Forms Fallback
-        if wikt and wikt.get("forms_list"):
-            forms_str = ", ".join([f['form_text'] for f in wikt['forms_list'][:5]])
-            html += f"<tr><td class='inflection-label'>Other Forms (Wikt)</td><td>{forms_str}...</td></tr>"
+        forms_list = inf_wikt.get("forms_list") or []
+        if forms_list:
+            # Safe extraction of form_text
+            forms_str_list = [f.get('form_text', '') for f in forms_list[:5] if f.get('form_text')]
+            if forms_str_list:
+                html += f"<tr><td class='inflection-label'>Other Forms (Wikt)</td><td>{', '.join(forms_str_list)}...</td></tr>"
 
         html += "</table></div>"
         
         # --- Semantics Section ---
-        sem = entry.get("semantics_combined", {})
         html += "<div class='ling-section'><div class='ling-subtitle'>Definitions & Senses</div>"
         
         # Wiktionary Senses
-        wikt_senses = sem.get("wiktionary_senses", [])
-        for s in wikt_senses[:3]: # Limit to top 3 for UI
-            gloss = s.get("definition", "").replace(";", "<br>")
-            html += f"<div class='sense-item'><span class='source-badge src-wikt'>Wikt</span> {gloss}</div>"
+        wikt_senses = sem_comb.get("wiktionary_senses") or []
+        for s in wikt_senses[:3]: 
+            # FIX: Ensure definition is a string before replacing
+            gloss_raw = s.get("definition") or "" 
+            gloss = gloss_raw.replace(";", "<br>")
+            if gloss:
+                html += f"<div class='sense-item'><span class='source-badge src-wikt'>Wikt</span> {gloss}</div>"
             
         # OEWN Senses
-        oewn_senses = sem.get("odenet_senses", [])
+        oewn_senses = sem_comb.get("odenet_senses") or []
         for s in oewn_senses[:3]:
-            defi = s.get("definition", "")
-            html += f"<div class='sense-item'><span class='source-badge src-oewn'>OEWN</span> {defi}</div>"
+            defi = s.get("definition") or ""
+            if defi:
+                html += f"<div class='sense-item'><span class='source-badge src-oewn'>OEWN</span> {defi}</div>"
             
         html += "</div>"
         
         # --- Relations Section (ConceptNet) ---
-        rels = sem.get("conceptnet_relations", [])
+        rels = sem_comb.get("conceptnet_relations") or []
         if rels:
             html += "<div class='ling-section'><div class='ling-subtitle'>Knowledge Graph (Top 5)</div>"
             html += "<div>"
             for r in rels[:5]:
                 rel_name = r.get("relation", "Related")
-                target = r.get("other_node") or r.get("surface")
+                # Safe extraction of target
+                target = r.get("other_node") or r.get("surface") or "?"
                 html += f"<span class='rel-chip'><span class='rel-type'>{rel_name}:</span> {target}</span>"
             html += "</div></div>"
             
@@ -2826,6 +2875,7 @@ if __name__ == "__main__":
              print("✗ WARNING: Failed to download English Wiktionary DB. Primary engine is disabled.")
         else:
              _ = wiktionary_get_connection() # Pre-warm
+             wiktionary_run_startup_diagnostics()
     except Exception as e:
         print(f"✗ FAILED to initialize Wiktionary: {e}")
     print("--- Wiktionary Done ---\n")