Update app.py

app.py

@@ -215,7 +215,7 @@ def log(msg):
         print(f"[DEBUG] {msg}")
 
 # --- Wiktionary Cache & Lock ---
-WIKTIONARY_DB_PATH = "de_wiktionary_normalized.db"
+WIKTIONARY_DB_PATH = "de_wiktionary_normalized_full.db"
 WIKTIONARY_REPO_ID = "cstr/de-wiktionary-sqlite-full"
 WIKTIONARY_CONN: Optional[sqlite3.Connection] = None
 WIKTIONARY_CONN_LOCK = threading.Lock()
@@ -1804,33 +1804,31 @@ def _wiktionary_map_pos_key(wikt_pos: Optional[str]) -> str:
 
 def _wiktionary_build_report_for_entry(entry_id: int, conn: sqlite3.Connection) -> Dict[str, Any]:
     """
-    (REVISED)
+    (REVISED FOR FULL DB)
     Fetches ALL associated data for a single Wiktionary entry_id,
-    making full use of the normalized norm4.py schema.
+    making full use of the new '_full' normalized schema.
     """
     report = {}
     
-    # 1. Get Base Entry Info
+    # 1. Get Base Entry Info (with title and redirect)
     entry_data = conn.execute(
-        "SELECT word, pos, pos_title, lang, etymology_text FROM entries WHERE id = ?", (entry_id,)
+        "SELECT word, title, redirect, pos, pos_title, lang, etymology_text FROM entries WHERE id = ?", (entry_id,)
     ).fetchone()
     if not entry_data:
         return {"error": "Entry ID not found"}
     report.update(dict(entry_data))
     report["entry_id"] = entry_id
-    report["lemma"] = entry_data["word"] # Alias for clarity
+    report["lemma"] = entry_data["word"]
 
-    # 2. Get Senses (with Glosses, Tags, Topics, Categories, and Examples)
+    # 2. Get Senses (with Glosses, Tags, Topics, Categories, Examples, Expressions, and Proverbs)
     senses_q = conn.execute(
         """
         SELECT 
             s.id as sense_id, 
             s.sense_index,
             (SELECT GROUP_CONCAT(g.gloss_text, '; ') FROM glosses g WHERE g.sense_id = s.id) as glosses,
-            (SELECT GROUP_CONCAT(rg.raw_gloss, '; ') FROM raw_glosses rg WHERE rg.sense_id = s.id) as raw_glosses,
             (SELECT GROUP_CONCAT(t.tag, ', ') FROM sense_tags st JOIN tags t ON st.tag_id = t.id WHERE st.sense_id = s.id) as tags,
-            (SELECT GROUP_CONCAT(top.topic, ', ') FROM sense_topics stop JOIN topics top ON stop.topic_id = top.id WHERE stop.sense_id = s.id) as topics,
-            (SELECT GROUP_CONCAT(c.category, ', ') FROM sense_categories sc JOIN categories c ON sc.category_id = c.id WHERE sc.sense_id = s.id) as categories
+            (SELECT GROUP_CONCAT(top.topic, ', ') FROM sense_topics stop JOIN topics top ON stop.topic_id = top.id WHERE stop.sense_id = s.id) as topics
         FROM senses s
         WHERE s.entry_id = ?
         ORDER BY s.id
@@ -1840,13 +1838,30 @@ def _wiktionary_build_report_for_entry(entry_id: int, conn: sqlite3.Connection)
     senses_list = []
     for sense_row in senses_q:
         sense_dict = dict(sense_row)
-        # Get examples for this specific sense
+        sense_id = sense_dict["sense_id"]
+        
+        # Get examples
         examples_q = conn.execute(
-            "SELECT text, ref, author, title, year, url FROM examples WHERE sense_id = ?", (sense_dict["sense_id"],)
+            "SELECT text, ref, author, title, year, url FROM examples WHERE sense_id = ?", (sense_id,)
         ).fetchall()
         sense_dict["examples"] = [dict(ex) for ex in examples_q]
+        
+        # --- NEW: Get expressions ---
+        expr_q = conn.execute(
+            "SELECT expression, sense_index FROM expressions WHERE sense_id = ?", (sense_id,)
+        ).fetchall()
+        sense_dict["expressions"] = [dict(ex) for ex in expr_q]
+        
+        # --- NEW: Get proverbs ---
+        prov_q = conn.execute(
+            "SELECT proverb, sense_index FROM proverbs WHERE sense_id = ?", (sense_id,)
+        ).fetchall()
+        sense_dict["proverbs"] = [dict(p) for p in prov_q]
+        
         senses_list.append(sense_dict)
     report["senses"] = senses_list
+    report["expressions_by_sense"] = [s["expressions"] for s in senses_list if s["expressions"]]
+    report["proverbs_by_sense"] = [s["proverbs"] for s in senses_list if s["proverbs"]]
 
     # 3. Get Inflected Forms (with Tags and Topics)
     forms_q = conn.execute(
@@ -1858,8 +1873,7 @@ def _wiktionary_build_report_for_entry(entry_id: int, conn: sqlite3.Connection)
             (SELECT GROUP_CONCAT(top.topic, ', ') FROM form_topics ftop JOIN topics top ON ftop.topic_id = top.id WHERE ftop.form_id = f.id) as topics
         FROM forms f
         WHERE f.entry_id = ?
-        GROUP BY f.id
-        ORDER BY f.id
+        GROUP BY f.id ORDER BY f.id
         """, (entry_id,)
     ).fetchall()
     report["forms"] = [dict(f) for f in forms_q]
@@ -1925,14 +1939,15 @@ def _wiktionary_build_report_for_entry(entry_id: int, conn: sqlite3.Connection)
     
     # 9. Get Derived and Related Terms
     derived_q = conn.execute(
-        "SELECT derived_word FROM derived_terms WHERE entry_id = ?", (entry_id,)
+        "SELECT derived_word, sense_index FROM derived_terms WHERE entry_id = ?", (entry_id,)
     ).fetchall()
-    report["derived_terms"] = [d["derived_word"] for d in derived_q]
+    report["derived_terms"] = [dict(d) for d in derived_q]
     
+    # --- FIXED: Query related_terms with new columns ---
     related_q = conn.execute(
-        "SELECT related_word FROM related_terms WHERE entry_id = ?", (entry_id,)
+        "SELECT related_word, sense_index, raw_tags_json FROM related_terms WHERE entry_id = ?", (entry_id,)
     ).fetchall()
-    report["related_terms"] = [r["related_word"] for r in related_q]
+    report["related_terms"] = [dict(r) for r in related_q]
     
     # 10. Get Entry-level Tags and Categories
     entry_tags_q = conn.execute(
@@ -1945,6 +1960,50 @@ def _wiktionary_build_report_for_entry(entry_id: int, conn: sqlite3.Connection)
     ).fetchall()
     report["entry_categories"] = [c["category"] for c in entry_cats_q]
 
+    # --- 11. GET ALL NEW OMITTED FIELDS ---
+    
+    # entry_notes
+    notes_q = conn.execute("SELECT note FROM entry_notes WHERE entry_id = ?", (entry_id,)).fetchall()
+    report["entry_notes"] = [n["note"] for n in notes_q]
+    
+    # other_pos
+    other_pos_q = conn.execute("SELECT pos_value FROM other_pos WHERE entry_id = ?", (entry_id,)).fetchall()
+    report["other_pos"] = [p["pos_value"] for p in other_pos_q]
+    
+    # entry_raw_tags
+    raw_tags_q = conn.execute("SELECT raw_tag FROM entry_raw_tags WHERE entry_id = ?", (entry_id,)).fetchall()
+    report["raw_tags"] = [t["raw_tag"] for t in raw_tags_q]
+    
+    # descendants
+    desc_q = conn.execute("SELECT lang, word, roman FROM descendants WHERE entry_id = ?", (entry_id,)).fetchall()
+    report["descendants"] = [dict(d) for d in desc_q]
+    
+    # Semantic relations
+    hyper_q = conn.execute("SELECT hypernym_word, sense_index FROM hypernyms WHERE entry_id = ?", (entry_id,)).fetchall()
+    report["hypernyms"] = [dict(h) for h in hyper_q]
+    
+    hypo_q = conn.execute("SELECT hyponym_word, sense_index FROM hyponyms WHERE entry_id = ?", (entry_id,)).fetchall()
+    report["hyponyms"] = [dict(h) for h in hypo_q]
+    
+    holo_q = conn.execute("SELECT holonym_word, sense_index FROM holonyms WHERE entry_id = ?", (entry_id,)).fetchall()
+    report["holonyms"] = [dict(h) for h in holo_q]
+    
+    mero_q = conn.execute("SELECT meronym_word, sense_index FROM meronyms WHERE entry_id = ?", (entry_id,)).fetchall()
+    report["meronyms"] = [dict(m) for m in mero_q]
+    
+    # Coordinate terms (with their tags)
+    coord_q = conn.execute(
+        """
+        SELECT 
+            ct.id, ct.coordinate_word, ct.sense_index,
+            (SELECT GROUP_CONCAT(t.tag, ', ') FROM coordinate_term_tags ctt JOIN tags t ON ctt.tag_id = t.id WHERE ctt.coordinate_term_id = ct.id) as tags
+        FROM coordinate_terms ct
+        WHERE ct.entry_id = ?
+        GROUP BY ct.id
+        """, (entry_id,)
+    ).fetchall()
+    report["coordinate_terms"] = [dict(c) for c in coord_q]
+
     return report
 
 def _wiktionary_find_all_entries(word: str, conn: sqlite3.Connection) -> List[Dict[str, Any]]:
@@ -2191,13 +2250,27 @@ def _analyze_word_with_wiktionary(word: str, top_n: int) -> Dict[str, Any]:
             "inflections_pattern": pattern_block,
             "semantics_combined": semantics_block,
             "wiktionary_metadata": {
+                # --- Original Fields ---
                 "pos_title": pos_title,
                 "etymology": wikt_report.get("etymology_text"),
                 "pronunciation": wikt_report.get("sounds"),
-                "hyphenation": wikt_report.get("hyphenations"), # <-- THE FIX
+                "hyphenation": wikt_report.get("hyphenations"),
                 "examples": [ex for s in wikt_report.get("senses", []) for ex in s.get("examples", [])],
                 "entry_tags": wikt_report.get("entry_tags"),
-                "entry_categories": wikt_report.get("entry_categories")
+                "entry_categories": wikt_report.get("entry_categories"),
+                
+                # --- NEW: Pass through all new fields from the full DB ---
+                "entry_notes": wikt_report.get("entry_notes"),
+                "other_pos": wikt_report.get("other_pos"),
+                "raw_tags": wikt_report.get("raw_tags"),
+                "descendants": wikt_report.get("descendants"),
+                "hypernyms": wikt_report.get("hypernyms"),
+                "hyponyms": wikt_report.get("hyponyms"),
+                "holonyms": wikt_report.get("holonyms"),
+                "meronyms": wikt_report.get("meronyms"),
+                "coordinate_terms": wikt_report.get("coordinate_terms"),
+                "expressions_by_sense": wikt_report.get("expressions_by_sense"),
+                "proverbs_by_sense": wikt_report.get("proverbs_by_sense")
             }
         }