app.py

import gradio as gr
import sqlite3
import pandas as pd
from huggingface_hub import hf_hub_download, HfApi
import os
import time
import json

# ===== CONFIGURATION =====
TARGET_LANGUAGES = ['de', 'en', 'es', 'fr', 'it', 'ja', 'nl', 'pl', 'pt', 'ru', 'zh']
INDEXED_REPO_ID = "cstr/conceptnet-de-indexed"
INDEXED_DB_FILENAME = "conceptnet-de-indexed.db"
PROGRESS_FILENAME = "indexing_progress.json"
LOCAL_DB_PATH = "/tmp/conceptnet-indexed.db"
CONCEPTNET_BASE = "http://conceptnet.io"
# =========================

print(f"🌍 Languages: {', '.join([l.upper() for l in TARGET_LANGUAGES])}")

HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN") or os.environ.get("HF_API_TOKEN")

if HF_TOKEN:
    print(f"✅ HF_TOKEN found")

ORIGINAL_REPO_ID = "ysenarath/conceptnet-sqlite"
ORIGINAL_DB_FILENAME = "data/conceptnet-v5.7.0.db"

def log_progress(message, level="INFO"):
    timestamp = time.strftime("%H:%M:%S")
    prefix = {"INFO": "ℹ️ ", "SUCCESS": "✅", "ERROR": "❌", "WARN": "⚠️ ", "DEBUG": "🔍"}.get(level, "")
    print(f"[{timestamp}] {prefix} {message}")

def check_remote_progress():
    if not HF_TOKEN:
        return {"indexing_complete": False}
    try:
        api = HfApi()
        api.repo_info(repo_id=INDEXED_REPO_ID, repo_type="dataset", token=HF_TOKEN)
        progress_path = hf_hub_download(repo_id=INDEXED_REPO_ID, filename=PROGRESS_FILENAME, repo_type="dataset", token=HF_TOKEN)
        with open(progress_path, 'r') as f:
            return json.load(f)
    except:
        return {"indexing_complete": False}

def create_indexed_database():
    progress = check_remote_progress()
    if progress.get("indexing_complete", False):
        try:
            indexed_path = hf_hub_download(repo_id=INDEXED_REPO_ID, filename=INDEXED_DB_FILENAME, repo_type="dataset", token=HF_TOKEN)
            log_progress("Downloaded indexed DB", "SUCCESS")
            return indexed_path
        except:
            pass
    return LOCAL_DB_PATH

DB_PATH = create_indexed_database()

def get_db_connection():
    conn = sqlite3.connect(DB_PATH, check_same_thread=False)
    conn.execute("PRAGMA cache_size = -256000")
    return conn

def deep_debug():
    """DEEP DEBUGGING - Find out what's actually wrong!"""
    log_progress("="*60, "INFO")
    log_progress("DEEP DEBUGGING SESSION", "INFO")
    log_progress("="*60, "INFO")
    
    try:
        with get_db_connection() as conn:
            cursor = conn.cursor()
            
            # 1. Find actual dog edges
            log_progress("\n1. Finding actual edges for 'dog':", "INFO")
            cursor.execute("""
                SELECT e.id, e.start_id, e.rel_id, e.end_id, e.weight
                FROM edge e
                WHERE e.start_id LIKE 'http://conceptnet.io/c/en/dog%'
                LIMIT 5
            """)
            edges = cursor.fetchall()
            
            log_progress(f"Found {len(edges)} edges:", "SUCCESS")
            for edge_id, start_id, rel_id, end_id, weight in edges:
                print(f"  {edge_id}")
                print(f"    start: {start_id}")
                print(f"    rel:   {rel_id}")
                print(f"    end:   {end_id}")
                print(f"    weight: {weight}")
            
            if not edges:
                log_progress("NO EDGES FOUND! Database might be corrupted!", "ERROR")
                return
            
            # 2. Check what relations actually exist
            log_progress("\n2. What relations exist?", "INFO")
            cursor.execute("SELECT id, label FROM relation LIMIT 20")
            relations = cursor.fetchall()
            log_progress(f"Found {len(relations)} relations:", "SUCCESS")
            for rel_id, label in relations:
                print(f"  {rel_id} -> {label}")
            
            # 3. Check if relation JOIN works
            log_progress("\n3. Testing relation JOIN:", "INFO")
            test_rel_id = edges[0][2] if edges else None
            if test_rel_id:
                log_progress(f"Looking up relation ID: {test_rel_id}", "DEBUG")
                cursor.execute("SELECT id, label FROM relation WHERE id = ?", (test_rel_id,))
                rel_result = cursor.fetchone()
                if rel_result:
                    log_progress(f"  ✅ Found: {rel_result[0]} -> {rel_result[1]}", "SUCCESS")
                else:
                    log_progress(f"  ❌ Relation ID not found in relation table!", "ERROR")
            
            # 4. Test the FULL JOIN query on ONE edge
            if edges:
                test_start = edges[0][1]
                log_progress(f"\n4. Testing full JOIN on: {test_start}", "INFO")
                
                query = """
                    SELECT 
                        e.id,
                        s.label AS start_label,
                        r.label AS relation,
                        en.label AS end_label,
                        e.weight
                    FROM edge e
                    JOIN node s ON e.start_id = s.id
                    JOIN relation r ON e.rel_id = r.id
                    JOIN node en ON e.end_id = en.id
                    WHERE e.start_id = ?
                    LIMIT 5
                """
                
                start = time.time()
                cursor.execute(query, (test_start,))
                results = cursor.fetchall()
                elapsed = time.time() - start
                
                log_progress(f"Full JOIN returned {len(results)} in {elapsed:.3f}s", "SUCCESS" if results else "ERROR")
                
                if results:
                    for edge_id, s_label, r_label, e_label, weight in results:
                        print(f"  {s_label} --{r_label}--> {e_label} [{weight:.3f}]")
                else:
                    log_progress("JOIN returned nothing! Checking each table...", "ERROR")
                    
                    # Debug each join
                    cursor.execute("SELECT id, label FROM node WHERE id = ?", (test_start,))
                    start_node = cursor.fetchone()
                    log_progress(f"  Start node: {start_node}", "DEBUG")
                    
                    test_end = edges[0][3]
                    cursor.execute("SELECT id, label FROM node WHERE id = ?", (test_end,))
                    end_node = cursor.fetchone()
                    log_progress(f"  End node: {end_node}", "DEBUG")
                    
                    test_rel = edges[0][2]
                    cursor.execute("SELECT id, label FROM relation WHERE id = ?", (test_rel,))
                    rel = cursor.fetchone()
                    log_progress(f"  Relation: {rel}", "DEBUG")
            
            # 5. Test with LIKE and JOIN
            log_progress("\n5. Testing LIKE + JOIN (what semantic profile does):", "INFO")
            
            test_pattern = f"{CONCEPTNET_BASE}/c/en/dog%"
            test_relation = "/r/IsA"
            
            query = """
                SELECT 
                    en.label,
                    e.weight
                FROM edge e
                JOIN node en ON e.end_id = en.id
                JOIN relation r ON e.rel_id = r.id
                WHERE e.start_id LIKE ? AND r.label = ?
                LIMIT 5
            """
            
            log_progress(f"Pattern: {test_pattern}", "DEBUG")
            log_progress(f"Relation: {test_relation}", "DEBUG")
            
            start = time.time()
            cursor.execute(query, (test_pattern, test_relation))
            results = cursor.fetchall()
            elapsed = time.time() - start
            
            log_progress(f"Result: {len(results)} rows in {elapsed:.3f}s", "SUCCESS" if results else "WARN")
            
            if results:
                for label, weight in results:
                    print(f"  dog IsA {label} [{weight:.3f}]")
            else:
                log_progress("No results! Let's check why...", "WARN")
                
                # Check if edges exist with this pattern
                cursor.execute("SELECT COUNT(*) FROM edge WHERE start_id LIKE ?", (test_pattern,))
                edge_count = cursor.fetchone()[0]
                log_progress(f"  Edges with pattern: {edge_count}", "DEBUG")
                
                # Check if any edges have this relation
                cursor.execute("SELECT COUNT(*) FROM edge e JOIN relation r ON e.rel_id = r.id WHERE r.label = ?", (test_relation,))
                rel_edge_count = cursor.fetchone()[0]
                log_progress(f"  Edges with relation {test_relation}: {rel_edge_count}", "DEBUG")
                
                # Check if the combination exists
                cursor.execute("""
                    SELECT COUNT(*) FROM edge e 
                    JOIN relation r ON e.rel_id = r.id 
                    WHERE e.start_id LIKE ? AND r.label = ?
                """, (test_pattern, test_relation))
                combo_count = cursor.fetchone()[0]
                log_progress(f"  Combination: {combo_count}", "DEBUG")
                
                if combo_count == 0:
                    log_progress("  ❌ NO edges match pattern + relation!", "ERROR")
                    log_progress("  Checking what relations DO exist for 'dog':", "INFO")
                    
                    cursor.execute("""
                        SELECT DISTINCT r.label, COUNT(*) as cnt
                        FROM edge e
                        JOIN relation r ON e.rel_id = r.id
                        WHERE e.start_id LIKE ?
                        GROUP BY r.label
                        ORDER BY cnt DESC
                        LIMIT 10
                    """, (test_pattern,))
                    
                    actual_rels = cursor.fetchall()
                    log_progress(f"  Actual relations for 'dog':", "INFO")
                    for rel_label, count in actual_rels:
                        print(f"    {rel_label}: {count} edges")
            
            log_progress("\n" + "="*60, "INFO")
            log_progress("DEBUGGING COMPLETE", "INFO")
            log_progress("="*60 + "\n", "INFO")
            
    except Exception as e:
        log_progress(f"Debug failed: {e}", "ERROR")
        import traceback
        traceback.print_exc()

# Run deep debugging
deep_debug()

def get_semantic_profile(word, lang='en', progress=gr.Progress()):
    """Semantic profile - will be fixed after we understand the debug output"""
    log_progress(f"Profile request: {word} ({lang})", "INFO")
    
    if not word or lang not in TARGET_LANGUAGES:
        return "⚠️ Invalid input"
    
    word = word.strip().lower().replace(' ', '_')
    like_path = f"{CONCEPTNET_BASE}/c/{lang}/{word}%"
    
    output_md = f"# 🧠 Semantic Profile: '{word}' ({lang.upper()})\n\n"
    output_md += "*Check server logs for detailed debug information*\n\n"
    
    try:
        with get_db_connection() as conn:
            cursor = conn.cursor()
            
            # Find nodes
            cursor.execute("SELECT id, label FROM node WHERE id LIKE ? LIMIT 5", (like_path,))
            nodes = cursor.fetchall()
            
            if not nodes:
                return f"# 🧠 '{word}'\n\n⚠️ Not found"
            
            for node_id, label in nodes[:3]:
                output_md += f"**Node:** `{node_id}` → {label}\n"
            
            output_md += "\n## Relations Found\n\n"
            
            # Get actual relations that exist
            query = """
                SELECT DISTINCT r.label, COUNT(*) as cnt
                FROM edge e
                JOIN relation r ON e.rel_id = r.id
                WHERE e.start_id LIKE ?
                GROUP BY r.label
                ORDER BY cnt DESC
            """
            
            cursor.execute(query, (like_path,))
            relations = cursor.fetchall()
            
            log_progress(f"Found {len(relations)} relation types", "INFO")
            
            for rel_label, count in relations[:20]:
                output_md += f"### {rel_label} ({count} edges)\n\n"
                
                # Get sample edges
                cursor.execute("""
                    SELECT en.label, e.weight
                    FROM edge e
                    JOIN node en ON e.end_id = en.id
                    JOIN relation r ON e.rel_id = r.id
                    WHERE e.start_id LIKE ? AND r.label = ?
                    ORDER BY e.weight DESC
                    LIMIT 5
                """, (like_path, rel_label))
                
                results = cursor.fetchall()
                for label, weight in results:
                    output_md += f"- **{word}** {rel_label} → *{label}* `[{weight:.3f}]`\n"
                
                output_md += "\n"
            
            return output_md
            
    except Exception as e:
        log_progress(f"Error: {e}", "ERROR")
        import traceback
        traceback.print_exc()
        return f"**❌ Error:** {e}"

def run_raw_query(sql_query):
    if not sql_query.strip().upper().startswith("SELECT"):
        return pd.DataFrame(), "❌ Only SELECT"
    try:
        with get_db_connection() as conn:
            start = time.time()
            df = pd.read_sql_query(sql_query, conn)
            elapsed = time.time() - start
            return df, f"✅ {len(df)} rows in {elapsed:.3f}s"
    except Exception as e:
        return pd.DataFrame(), f"❌ {e}"

def get_schema_info():
    return f"# Schema\n\nCheck server logs for detailed debugging output."

# UI
with gr.Blocks(title="ConceptNet Debug", theme=gr.themes.Soft()) as demo:
    gr.Markdown("# 🔍 ConceptNet Debugger")
    gr.Markdown("**Check server logs for comprehensive debugging information!**")
    
    with gr.Tabs():
        with gr.TabItem("🔍 Profile"):
            with gr.Row():
                word_input = gr.Textbox(label="Word", value="dog")
                lang_input = gr.Dropdown(choices=TARGET_LANGUAGES, value="en", label="Lang")
            profile_btn = gr.Button("Get Profile")
            profile_out = gr.Markdown()
        
        with gr.TabItem("💻 SQL"):
            sql_input = gr.Textbox(
                label="SQL",
                value="SELECT e.*, r.label FROM edge e JOIN relation r ON e.rel_id = r.id WHERE e.start_id LIKE 'http://conceptnet.io/c/en/dog%' LIMIT 10",
                lines=3
            )
            sql_btn = gr.Button("Execute")
            sql_status = gr.Markdown()
            sql_results = gr.DataFrame()
        
        with gr.TabItem("📊 Schema"):
            schema_btn = gr.Button("Load")
            schema_out = gr.Markdown()
    
    profile_btn.click(get_semantic_profile, [word_input, lang_input], profile_out)
    sql_btn.click(run_raw_query, sql_input, [sql_results, sql_status])
    schema_btn.click(get_schema_info, None, schema_out)

if __name__ == "__main__":
    log_progress("DEBUG MODE READY", "SUCCESS")
    demo.launch(ssr_mode=False)