gunzip wiktionary

app.py

@@ -50,6 +50,8 @@ import re
 import sqlite3
 import json
 from huggingface_hub import hf_hub_download
+import gzip
+import shutil
 
 # --- Requests and gradio Import (for ConceptNet) ---
 try:
@@ -225,8 +227,9 @@ def log(msg):
         print(f"[DEBUG] {msg}")
 
 # --- Wiktionary Cache & Lock (ENGLISH) ---
-WIKTIONARY_DB_PATH = "en_wiktionary_normalized.db"
 WIKTIONARY_REPO_ID = "cstr/en-wiktionary-sqlite-full"
+WIKTIONARY_REMOTE_FILE = "en_wiktionary_normalized_full.db.gz" # File as seen in your screenshot
+WIKTIONARY_DB_PATH = "en_wiktionary_normalized.db" # Local extracted file
 WIKTIONARY_CONN: Optional[sqlite3.Connection] = None
 WIKTIONARY_CONN_LOCK = threading.Lock()
 WIKTIONARY_AVAILABLE = False
@@ -1227,27 +1230,44 @@ def _analyze_word_with_hanta_en(word: str, top_n: int) -> Dict[str, Any]:
 # 6d. WIKTIONARY DATABASE LOGIC (EN)
 # ============================================================================
 def wiktionary_download_db() -> bool:
-    """ Downloads the English Wiktionary DB. """
+    """ Downloads the compressed English Wiktionary DB and extracts it. """
     global WIKTIONARY_AVAILABLE
+    
+    # Check if the extracted DB already exists
     if os.path.exists(WIKTIONARY_DB_PATH):
         print(f"✓ English Wiktionary DB '{WIKTIONARY_DB_PATH}' already exists.")
         WIKTIONARY_AVAILABLE = True
         return True
 
-    print(f"English Wiktionary DB not found. Downloading from '{WIKTIONARY_REPO_ID}'...")
+    print(f"English Wiktionary DB not found. Downloading '{WIKTIONARY_REMOTE_FILE}' from '{WIKTIONARY_REPO_ID}'...")
     try:
-        hf_hub_download(
+        # 1. Download the .gz file
+        downloaded_gz_path = hf_hub_download(
             repo_id=WIKTIONARY_REPO_ID,
-            filename=WIKTIONARY_DB_PATH,
+            filename=WIKTIONARY_REMOTE_FILE,
             repo_type="dataset",
-            local_dir=".",
-            local_dir_use_symlinks=False
+            local_dir="."
+            # Removed deprecated `local_dir_use_symlinks`
         )
-        print(f"✓ English Wiktionary DB downloaded successfully.")
+        
+        # 2. Decompress the .gz file to the .db file
+        print(f"Downloading complete. Extracting '{downloaded_gz_path}' to '{WIKTIONARY_DB_PATH}'...")
+        with gzip.open(downloaded_gz_path, 'rb') as f_in:
+            with open(WIKTIONARY_DB_PATH, 'wb') as f_out:
+                shutil.copyfileobj(f_in, f_out)
+        
+        # Optional: Cleanup the .gz file to save space
+        try:
+            os.remove(downloaded_gz_path)
+        except OSError:
+            pass
+
+        print(f"✓ English Wiktionary DB downloaded and extracted successfully.")
         WIKTIONARY_AVAILABLE = True
         return True
     except Exception as e:
-        print(f"✗ CRITICAL: Failed to download English Wiktionary DB: {e}")
+        print(f"✗ CRITICAL: Failed to download/extract English Wiktionary DB: {e}")
+        # traceback.print_exc() # Uncomment for deep debugging
         return False
 
 def wiktionary_get_connection() -> Optional[sqlite3.Connection]: