Update app.py

app.py

@@ -18,13 +18,12 @@ LOCAL_DB_PATH = "/tmp/conceptnet-indexed.db"
 
 print(f"🌍 Filtering to: {', '.join([l.upper() for l in TARGET_LANGUAGES])}")
 
-# Get HF token (multiple methods for compatibility)
+# Get HF token
 HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN") or os.environ.get("HF_API_TOKEN")
 
 if not HF_TOKEN:
     print("⚠️  WARNING: No HF_TOKEN found!")
     print("   Add HF_TOKEN in Space settings to enable checkpointing")
-    print("   Go to: Settings > Variables and secrets > HF_TOKEN")
 else:
     print(f"✅ HF_TOKEN found (length: {len(HF_TOKEN)})")
 
@@ -44,6 +43,30 @@ def log_progress(message, level="INFO"):
     }.get(level, "")
     print(f"[{timestamp}] {prefix} {message}")
 
+def verify_database_has_indices(db_path):
+    """
+    Verify that a database file actually has the required indices.
+    Returns (has_indices, index_count)
+    """
+    if not os.path.exists(db_path):
+        return False, 0
+    
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        
+        # Check for custom indices
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='index' AND name LIKE 'idx_%'")
+        custom_indices = cursor.fetchall()
+        
+        conn.close()
+        
+        return len(custom_indices) >= 4, len(custom_indices)
+        
+    except Exception as e:
+        log_progress(f"Error verifying indices: {e}", "ERROR")
+        return False, 0
+
 def check_remote_progress():
     """Check remote progress with detailed logging"""
     if not HF_TOKEN:
@@ -166,7 +189,7 @@ def update_remote_progress(completed_indices, analyzed_tables=None, database_upl
         return False
 
 def upload_database_checkpoint(message=""):
-    """Upload database with progress reporting"""
+    """Upload database with proper WAL checkpoint"""
     if not HF_TOKEN:
         log_progress("Cannot upload: No HF_TOKEN", "WARN")
         return False
@@ -176,6 +199,20 @@ def upload_database_checkpoint(message=""):
         return False
     
     try:
+        # CRITICAL FIX: Checkpoint WAL before upload
+        log_progress("Checkpointing WAL to merge changes into main file...", "INFO")
+        conn = sqlite3.connect(LOCAL_DB_PATH)
+        conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+        conn.close()
+        log_progress("  WAL checkpoint complete", "SUCCESS")
+        
+        # Verify indices are actually in the file
+        has_indices, idx_count = verify_database_has_indices(LOCAL_DB_PATH)
+        if has_indices:
+            log_progress(f"  Verified: {idx_count} indices present in file", "SUCCESS")
+        else:
+            log_progress(f"  WARNING: Only {idx_count} indices found (expected 4+)", "WARN")
+        
         api = HfApi()
         
         db_size = os.path.getsize(LOCAL_DB_PATH) / (2**30)
@@ -220,9 +257,9 @@ def create_indexed_database():
     database_uploaded = progress.get("database_uploaded", False)
     indexing_complete = progress.get("indexing_complete", False)
     
-    # If fully complete, download and return
+    # If fully complete, try to download
     if indexing_complete:
-        log_progress("Fully indexed database exists!", "SUCCESS")
+        log_progress("Fully indexed database exists in HF!", "SUCCESS")
         log_progress(f"Downloading from {INDEXED_REPO_ID}...", "INFO")
         
         try:
@@ -233,38 +270,41 @@ def create_indexed_database():
                 token=HF_TOKEN
             )
             log_progress(f"Downloaded: {indexed_path}", "SUCCESS")
-            return indexed_path
+            
+            # CRITICAL CHECK: Verify downloaded database actually has indices
+            log_progress("Verifying downloaded database has indices...", "INFO")
+            has_indices, idx_count = verify_database_has_indices(indexed_path)
+            
+            if has_indices:
+                log_progress(f"  ✅ Verified: {idx_count} indices found", "SUCCESS")
+                return indexed_path
+            else:
+                log_progress(f"  ❌ CORRUPTED: Only {idx_count} indices (expected 4+)", "ERROR")
+                log_progress("  The uploaded database is missing indices!", "ERROR")
+                log_progress("  This was caused by WAL mode not being checkpointed.", "ERROR")
+                log_progress("  Forcing re-indexing...", "WARN")
+                
+                # Reset progress to force rebuild
+                indexing_complete = False
+                completed_indices = set()
+                analyzed_tables = set()
+                database_uploaded = False
+                
+                # Update remote to clear the bad state
+                update_remote_progress([], [], database_uploaded=False, indexing_complete=False)
             
         except Exception as e:
             log_progress(f"Download failed: {e}", "ERROR")
             log_progress("Will create locally", "INFO")
     
-    # NEW: If all indices complete OR partial work done, try to download indexed DB
-    if completed_indices or analyzed_tables or database_uploaded:
-        log_progress("Checkpoint detected - attempting to download indexed database...", "INFO")
+    # Check for partial progress
+    if completed_indices or analyzed_tables:
+        log_progress("Resuming from checkpoint:", "INFO")
         log_progress(f"  Completed indices: {sorted(completed_indices)}", "INFO")
         log_progress(f"  Analyzed tables: {sorted(analyzed_tables)}", "INFO")
-        
-        try:
-            indexed_path = hf_hub_download(
-                repo_id=INDEXED_REPO_ID,
-                filename=INDEXED_DB_FILENAME,
-                repo_type="dataset",
-                token=HF_TOKEN
-            )
-            
-            # Copy to local path
-            log_progress(f"Downloaded indexed DB, copying to {LOCAL_DB_PATH}...", "INFO")
-            shutil.copy2(indexed_path, LOCAL_DB_PATH)
-            log_progress("Using downloaded indexed database", "SUCCESS")
-            
-        except Exception as e:
-            log_progress(f"Could not download indexed DB: {e}", "WARN")
-            log_progress("Will need to start from original and create indices", "INFO")
     
-    # Only download original if we don't have local DB yet
+    # Download original database if needed
     if not os.path.exists(LOCAL_DB_PATH):
-        # Download and copy original
         log_progress("Downloading original database...", "INFO")
         original_path = hf_hub_download(
             repo_id=ORIGINAL_REPO_ID,
@@ -286,17 +326,12 @@ def create_indexed_database():
         shutil.copy2(original_path, LOCAL_DB_PATH)
         elapsed = time.time() - start
         log_progress(f"Copied in {elapsed:.1f}s", "SUCCESS")
-        
-        # Clear completed indices since we're starting from scratch
-        log_progress("Starting from original DB - clearing completed indices list", "WARN")
-        completed_indices = set()
-        analyzed_tables = set()
     
     # Connect to database
     conn = sqlite3.connect(LOCAL_DB_PATH)
     cursor = conn.cursor()
     
-    # Enable optimizations
+    # Enable optimizations (but we'll checkpoint WAL before upload!)
     cursor.execute("PRAGMA journal_mode = WAL")
     cursor.execute("PRAGMA synchronous = NORMAL")
     cursor.execute("PRAGMA cache_size = -512000")
@@ -339,7 +374,7 @@ def create_indexed_database():
                 indexing_complete=False
             )
             
-            # Upload checkpoint
+            # Upload checkpoint (with WAL checkpoint!)
             upload_database_checkpoint(f"Checkpoint: {idx_name} created ({i}/{len(indices_to_create)})")
             
         except Exception as e:
@@ -347,12 +382,11 @@ def create_indexed_database():
             conn.close()
             raise
     
-    # PHASE 2: Analyze Tables (per-table with checkpoints)
+    # PHASE 2: Analyze Tables
     log_progress("="*60, "INFO")
     log_progress("PHASE 2: ANALYZING TABLES", "INFO")
     log_progress("="*60, "INFO")
     
-    # Get list of tables
     cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
     tables = [row[0] for row in cursor.fetchall()]
     
@@ -365,7 +399,6 @@ def create_indexed_database():
         
         log_progress(f"[{i}/{len(tables)}] Analyzing table: {table}", "INFO")
         
-        # Get table size
         try:
             cursor.execute(f"SELECT COUNT(*) FROM {table}")
             row_count = cursor.fetchone()[0]
@@ -376,7 +409,6 @@ def create_indexed_database():
         start = time.time()
         
         try:
-            # Run ANALYZE on this specific table
             cursor.execute(f"ANALYZE {table}")
             conn.commit()
             
@@ -392,7 +424,7 @@ def create_indexed_database():
                 indexing_complete=False
             )
             
-            # Upload checkpoint after each table
+            # Upload checkpoint
             log_progress(f"  Uploading checkpoint after analyzing {table}...", "CHECKPOINT")
             upload_database_checkpoint(f"Checkpoint: {table} analyzed ({i}/{len(tables)})")
             
@@ -400,17 +432,29 @@ def create_indexed_database():
             log_progress(f"Failed to analyze {table}: {e}", "ERROR")
             log_progress("Continuing with next table...", "WARN")
     
+    # CRITICAL: Final WAL checkpoint before closing
+    log_progress("Performing final WAL checkpoint...", "INFO")
+    cursor.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+    conn.commit()
     conn.close()
+    log_progress("  All changes merged to main database file", "SUCCESS")
     
-    # PHASE 3: Final upload and completion
+    # PHASE 3: Final upload
     log_progress("="*60, "INFO")
     log_progress("PHASE 3: FINAL UPLOAD", "INFO")
     log_progress("="*60, "INFO")
     
     log_progress("All indexing and analysis complete!", "SUCCESS")
-    log_progress("Performing final upload...", "INFO")
     
-    upload_database_checkpoint("Final indexed database - COMPLETE")
+    # Verify one more time before final upload
+    has_indices, idx_count = verify_database_has_indices(LOCAL_DB_PATH)
+    if has_indices:
+        log_progress(f"Final verification: {idx_count} indices confirmed in file ✅", "SUCCESS")
+    else:
+        log_progress(f"WARNING: Only {idx_count} indices in file!", "ERROR")
+    
+    log_progress("Performing final upload...", "INFO")
+    upload_database_checkpoint("Final indexed database - COMPLETE (with WAL checkpoint)")
     
     # Mark as complete
     update_remote_progress(
@@ -455,10 +499,13 @@ def verify_indices():
         cursor.execute("SELECT name FROM sqlite_master WHERE type='index' AND name LIKE 'idx_%'")
         custom_indices = cursor.fetchall()
         
-        log_progress(f"Custom indices: {len(custom_indices)}", "INFO")
+        log_progress(f"Custom indices: {len(custom_indices)}", "SUCCESS" if len(custom_indices) >= 4 else "ERROR")
         for idx in custom_indices:
             log_progress(f"  ✓ {idx[0]}", "SUCCESS")
         
+        if len(custom_indices) < 4:
+            log_progress("⚠️  WARNING: Expected 4 indices, something went wrong!", "ERROR")
+        
         # Speed test
         log_progress("Running speed test...", "INFO")
         start = time.time()
@@ -469,6 +516,9 @@ def verify_indices():
         status = "SUCCESS" if elapsed < 1 else "WARN" if elapsed < 5 else "ERROR"
         log_progress(f"Query: {count} results in {elapsed:.3f}s", status)
         
+        if elapsed > 5:
+            log_progress("⚠️  Query is slow - indices may not be working!", "ERROR")
+        
         log_progress("="*60, "INFO")
 
 verify_indices()
@@ -645,7 +695,7 @@ def get_schema_info():
 # UI
 with gr.Blocks(title="ConceptNet", theme=gr.themes.Soft()) as demo:
     gr.Markdown(f"# 🧠 ConceptNet ({', '.join([l.upper() for l in TARGET_LANGUAGES])})")
-    gr.Markdown(f"**Repo:** [{INDEXED_REPO_ID}](https://huggingface.co/datasets/{INDEXED_REPO_ID}) | ✅ Per-table checkpoints")
+    gr.Markdown(f"**Repo:** [{INDEXED_REPO_ID}](https://huggingface.co/datasets/{INDEXED_REPO_ID}) | ✅ Fixed WAL checkpoint issue")
     
     with gr.Tabs():
         with gr.TabItem("🔍 Profile"):
@@ -675,7 +725,7 @@ with gr.Blocks(title="ConceptNet", theme=gr.themes.Soft()) as demo:
             schema_btn = gr.Button("📊 Load")
             schema_output = gr.Markdown()
     
-    gr.Markdown("---\n✅ **Per-table ANALYZE with checkpoints!** Check server logs for detailed progress.")
+    gr.Markdown("---\n✅ **Fixed WAL checkpoint issue!** Database now properly contains indices. Will auto-rebuild if corrupted DB detected.")
     
     semantic_btn.click(get_semantic_profile, [word_input, lang_input], semantic_output)
     query_btn.click(run_query, [start_input, rel_input, end_input, limit_slider], [results_output, status_output])
@@ -683,5 +733,5 @@ with gr.Blocks(title="ConceptNet", theme=gr.themes.Soft()) as demo:
     schema_btn.click(get_schema_info, None, schema_output)
 
 if __name__ == "__main__":
-    log_progress("App ready with per-table ANALYZE checkpoints!", "SUCCESS")
+    log_progress("App ready with WAL checkpoint fix!", "SUCCESS")
     demo.launch(ssr_mode=False)