Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -4,47 +4,185 @@ import pandas as pd
|
|
| 4 |
from huggingface_hub import hf_hub_download, snapshot_download
|
| 5 |
import os
|
| 6 |
import time
|
|
|
|
| 7 |
from pathlib import Path
|
| 8 |
|
| 9 |
# ===== CONFIGURATION =====
|
| 10 |
TARGET_LANGUAGES = ['de']
|
|
|
|
| 11 |
# =========================
|
| 12 |
|
| 13 |
print(f"π Filtering to: {', '.join([l.upper() for l in TARGET_LANGUAGES])}")
|
| 14 |
|
| 15 |
-
# Download database
|
| 16 |
REPO_ID = "ysenarath/conceptnet-sqlite"
|
| 17 |
DB_FILENAME = "data/conceptnet-v5.7.0.db"
|
| 18 |
|
| 19 |
-
|
| 20 |
-
print(f"
|
| 21 |
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
)
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
|
| 34 |
def get_db_connection():
|
| 35 |
-
"""Create optimized connection"""
|
| 36 |
-
|
| 37 |
-
conn = sqlite3.connect(db_uri, uri=True, check_same_thread=False)
|
| 38 |
-
conn.execute("PRAGMA query_only = ON")
|
| 39 |
conn.execute("PRAGMA cache_size = -256000")
|
| 40 |
conn.execute("PRAGMA mmap_size = 4294967296")
|
| 41 |
conn.execute("PRAGMA temp_store = MEMORY")
|
| 42 |
return conn
|
| 43 |
|
| 44 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
"""
|
| 46 |
-
|
| 47 |
-
This avoids full table scan on edge table.
|
| 48 |
"""
|
| 49 |
if not word:
|
| 50 |
return "β οΈ Please enter a word."
|
|
@@ -55,10 +193,6 @@ def get_semantic_profile_fast(word, lang='de'):
|
|
| 55 |
word = word.strip().lower().replace(' ', '_')
|
| 56 |
like_path = f"/c/{lang}/{word}%"
|
| 57 |
|
| 58 |
-
print(f"\n{'='*60}")
|
| 59 |
-
print(f"Semantic Profile: {word} ({lang})")
|
| 60 |
-
print(f"{'='*60}")
|
| 61 |
-
|
| 62 |
relations = [
|
| 63 |
"/r/IsA", "/r/PartOf", "/r/HasA", "/r/UsedFor", "/r/CapableOf",
|
| 64 |
"/r/Causes", "/r/HasProperty", "/r/Synonym", "/r/Antonym",
|
|
@@ -71,90 +205,61 @@ def get_semantic_profile_fast(word, lang='de'):
|
|
| 71 |
with get_db_connection() as conn:
|
| 72 |
cursor = conn.cursor()
|
| 73 |
|
| 74 |
-
#
|
| 75 |
-
print(f"Step 1: Finding nodes matching '{like_path}'...")
|
| 76 |
-
start = time.time()
|
| 77 |
cursor.execute("SELECT id, label FROM node WHERE id LIKE ?", (like_path,))
|
| 78 |
-
|
| 79 |
-
elapsed = time.time() - start
|
| 80 |
-
print(f" Found {len(matching_nodes)} nodes in {elapsed:.3f}s")
|
| 81 |
-
|
| 82 |
-
if not matching_nodes:
|
| 83 |
-
return f"# π§ Semantic Profile: '{word}'\n\nβ οΈ **No nodes found**\n\nTry checking spelling or use a more common word."
|
| 84 |
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
primary_id = node_ids[0]
|
| 88 |
-
print(f" Primary ID: {primary_id}")
|
| 89 |
|
| 90 |
-
for node_id, label in
|
| 91 |
output_md += f"**Node:** `{node_id}` ({label})\n"
|
| 92 |
output_md += "\n"
|
| 93 |
|
| 94 |
total_relations = 0
|
| 95 |
|
| 96 |
-
#
|
| 97 |
for rel in relations:
|
| 98 |
-
print(f"\nStep 2: Querying {rel}...")
|
| 99 |
output_md += f"## {rel}\n\n"
|
| 100 |
-
|
| 101 |
has_results = False
|
| 102 |
|
| 103 |
-
# Outgoing edges - FAST
|
| 104 |
-
|
| 105 |
-
# Use IN with explicit node IDs - much faster than LIKE on edge table
|
| 106 |
-
placeholders = ','.join(['?'] * len(node_ids))
|
| 107 |
-
query_out = f"""
|
| 108 |
SELECT en.label, e.weight
|
| 109 |
FROM edge e
|
| 110 |
JOIN node en ON e.end_id = en.id
|
| 111 |
JOIN relation r ON e.rel_id = r.id
|
| 112 |
-
WHERE e.start_id
|
| 113 |
ORDER BY e.weight DESC
|
| 114 |
-
LIMIT
|
| 115 |
-
"""
|
| 116 |
-
cursor.execute(query_out, node_ids + [rel])
|
| 117 |
-
out_results = cursor.fetchall()
|
| 118 |
-
elapsed = time.time() - start
|
| 119 |
-
print(f" Outgoing: {len(out_results)} results in {elapsed:.3f}s")
|
| 120 |
|
| 121 |
-
for label, weight in
|
| 122 |
output_md += f"- **{word}** {rel} β *{label}* `[{weight:.3f}]`\n"
|
| 123 |
has_results = True
|
| 124 |
total_relations += 1
|
| 125 |
|
| 126 |
-
# Incoming edges
|
| 127 |
-
|
| 128 |
-
query_in = f"""
|
| 129 |
SELECT s.label, e.weight
|
| 130 |
FROM edge e
|
| 131 |
JOIN node s ON e.start_id = s.id
|
| 132 |
JOIN relation r ON e.rel_id = r.id
|
| 133 |
-
WHERE e.end_id
|
| 134 |
ORDER BY e.weight DESC
|
| 135 |
-
LIMIT
|
| 136 |
-
"""
|
| 137 |
-
cursor.execute(query_in, node_ids + [rel])
|
| 138 |
-
in_results = cursor.fetchall()
|
| 139 |
-
elapsed = time.time() - start
|
| 140 |
-
print(f" Incoming: {len(in_results)} results in {elapsed:.3f}s")
|
| 141 |
|
| 142 |
-
for label, weight in
|
| 143 |
output_md += f"- *{label}* {rel} β **{word}** `[{weight:.3f}]`\n"
|
| 144 |
has_results = True
|
| 145 |
total_relations += 1
|
| 146 |
|
| 147 |
if not has_results:
|
| 148 |
output_md += "*No results*\n"
|
| 149 |
-
|
| 150 |
output_md += "\n"
|
| 151 |
|
| 152 |
-
output_md += "---\n"
|
| 153 |
-
output_md += f"**Total relations found:** {total_relations}\n"
|
| 154 |
-
|
| 155 |
-
print(f"\nβ
Complete: {total_relations} relations")
|
| 156 |
-
print("="*60 + "\n")
|
| 157 |
-
|
| 158 |
return output_md
|
| 159 |
|
| 160 |
except Exception as e:
|
|
@@ -163,67 +268,38 @@ def get_semantic_profile_fast(word, lang='de'):
|
|
| 163 |
traceback.print_exc()
|
| 164 |
return f"**β Error:**\n\n```\n{e}\n```"
|
| 165 |
|
| 166 |
-
def
|
| 167 |
-
"""
|
| 168 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
"""
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
print(f"{'='*60}")
|
| 173 |
|
| 174 |
try:
|
| 175 |
with get_db_connection() as conn:
|
| 176 |
-
|
|
|
|
|
|
|
| 177 |
|
| 178 |
-
|
| 179 |
-
end_ids = []
|
| 180 |
-
|
| 181 |
-
# Step 1: Get start node IDs (if specified)
|
| 182 |
if start_node:
|
| 183 |
pattern = start_node if '%' in start_node else f"%{start_node}%"
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
print(f" Start nodes: {len(start_ids)}")
|
| 187 |
-
|
| 188 |
-
if not start_ids:
|
| 189 |
-
return pd.DataFrame(), f"No nodes found matching '{start_node}'"
|
| 190 |
-
|
| 191 |
-
# Step 2: Get end node IDs (if specified)
|
| 192 |
-
if end_node:
|
| 193 |
-
pattern = end_node if '%' in end_node else f"%{end_node}%"
|
| 194 |
-
cursor.execute("SELECT id FROM node WHERE id LIKE ? LIMIT 100", (pattern,))
|
| 195 |
-
end_ids = [row[0] for row in cursor.fetchall()]
|
| 196 |
-
print(f" End nodes: {len(end_ids)}")
|
| 197 |
-
|
| 198 |
-
if not end_ids:
|
| 199 |
-
return pd.DataFrame(), f"No nodes found matching '{end_node}'"
|
| 200 |
-
|
| 201 |
-
# Step 3: Query edges with exact ID matches
|
| 202 |
-
query = """
|
| 203 |
-
SELECT
|
| 204 |
-
e.id,
|
| 205 |
-
s.id,
|
| 206 |
-
r.label,
|
| 207 |
-
en.id,
|
| 208 |
-
e.weight,
|
| 209 |
-
s.label,
|
| 210 |
-
en.label
|
| 211 |
-
FROM edge e
|
| 212 |
-
JOIN relation r ON e.rel_id = r.id
|
| 213 |
-
JOIN node s ON e.start_id = s.id
|
| 214 |
-
JOIN node en ON e.end_id = en.id
|
| 215 |
-
WHERE 1=1
|
| 216 |
-
"""
|
| 217 |
-
params = []
|
| 218 |
-
|
| 219 |
-
# Add language filter with IN clause for speed
|
| 220 |
-
lang_ids_query = " OR ".join([f"s.id LIKE '/c/{lang}/%' OR en.id LIKE '/c/{lang}/%'" for lang in TARGET_LANGUAGES])
|
| 221 |
-
query += f" AND ({lang_ids_query})"
|
| 222 |
-
|
| 223 |
-
if start_ids:
|
| 224 |
-
placeholders = ','.join(['?'] * len(start_ids))
|
| 225 |
-
query += f" AND e.start_id IN ({placeholders})"
|
| 226 |
-
params.extend(start_ids)
|
| 227 |
|
| 228 |
if relation:
|
| 229 |
if '%' in relation:
|
|
@@ -232,25 +308,20 @@ def run_query_fast(start_node, relation, end_node, limit):
|
|
| 232 |
query += " AND r.label = ?"
|
| 233 |
params.append(relation)
|
| 234 |
|
| 235 |
-
if
|
| 236 |
-
|
| 237 |
-
query +=
|
| 238 |
-
params.
|
| 239 |
|
| 240 |
query += " ORDER BY e.weight DESC LIMIT ?"
|
| 241 |
params.append(limit)
|
| 242 |
|
| 243 |
-
print(f" Executing query with {len(params)} params...")
|
| 244 |
start_time = time.time()
|
| 245 |
-
|
| 246 |
df = pd.read_sql_query(query, conn, params=params)
|
| 247 |
-
|
| 248 |
elapsed = time.time() - start_time
|
| 249 |
-
print(f" β
{len(df)} results in {elapsed:.2f}s")
|
| 250 |
-
print("="*60 + "\n")
|
| 251 |
|
| 252 |
if df.empty:
|
| 253 |
-
return pd.DataFrame(), f"No results
|
| 254 |
|
| 255 |
df.columns = ['edge_id', 'start_id', 'relation', 'end_id', 'weight', 'start_label', 'end_label']
|
| 256 |
return df, f"β
{len(df)} results in {elapsed:.2f}s"
|
|
@@ -259,7 +330,7 @@ def run_query_fast(start_node, relation, end_node, limit):
|
|
| 259 |
print(f"ERROR: {e}")
|
| 260 |
import traceback
|
| 261 |
traceback.print_exc()
|
| 262 |
-
return pd.DataFrame(), f"**β Error
|
| 263 |
|
| 264 |
def run_raw_query(sql_query):
|
| 265 |
"""Execute raw SQL"""
|
|
@@ -276,14 +347,12 @@ def run_raw_query(sql_query):
|
|
| 276 |
return pd.DataFrame(), f"Error: {e}"
|
| 277 |
|
| 278 |
def get_schema_info():
|
| 279 |
-
"""Get schema"""
|
| 280 |
with get_db_connection() as conn:
|
| 281 |
cursor = conn.cursor()
|
| 282 |
|
| 283 |
-
md = "# π Schema\n\n"
|
| 284 |
-
md += "
|
| 285 |
-
md += "This means LIKE queries on edge table do full table scans (34M rows).\n\n"
|
| 286 |
-
md += "**Workaround:** Query node table first (has index), then use exact ID matches.\n\n"
|
| 287 |
|
| 288 |
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
|
| 289 |
tables = cursor.fetchall()
|
|
@@ -293,69 +362,47 @@ def get_schema_info():
|
|
| 293 |
count = cursor.fetchone()[0]
|
| 294 |
md += f"## {table_name} ({count:,} rows)\n\n"
|
| 295 |
|
|
|
|
| 296 |
cursor.execute(f"PRAGMA table_info({table_name})")
|
| 297 |
cols = cursor.fetchall()
|
| 298 |
-
|
| 299 |
-
md += "| Column | Type |\n|:--|:--|\n"
|
| 300 |
for col in cols:
|
| 301 |
-
md += f"| `{col[1]}` | `{col[2]}` |\n"
|
| 302 |
|
|
|
|
| 303 |
cursor.execute(f"PRAGMA index_list({table_name})")
|
| 304 |
indices = cursor.fetchall()
|
| 305 |
|
| 306 |
if indices:
|
| 307 |
-
md += f"\n**Indices
|
| 308 |
for idx in indices:
|
| 309 |
cursor.execute(f"PRAGMA index_info({idx[1]})")
|
| 310 |
idx_cols = cursor.fetchall()
|
| 311 |
-
cols_str = ', '.join([c[2] for c in idx_cols if c[2]]) or '
|
| 312 |
-
|
|
|
|
|
|
|
|
|
|
| 313 |
|
| 314 |
md += "\n---\n\n"
|
| 315 |
|
| 316 |
return md
|
| 317 |
|
| 318 |
-
# Test on startup
|
| 319 |
-
print("\nπ§ͺ TESTING DATABASE...")
|
| 320 |
-
with get_db_connection() as conn:
|
| 321 |
-
cursor = conn.cursor()
|
| 322 |
-
|
| 323 |
-
# Test 1: Node query (should be fast - has index)
|
| 324 |
-
start = time.time()
|
| 325 |
-
cursor.execute("SELECT COUNT(*) FROM node WHERE id LIKE '/c/de/%'")
|
| 326 |
-
de_count = cursor.fetchone()[0]
|
| 327 |
-
elapsed = time.time() - start
|
| 328 |
-
print(f"β
DE nodes: {de_count:,} ({elapsed:.3f}s)")
|
| 329 |
-
|
| 330 |
-
# Test 2: Get specific node
|
| 331 |
-
cursor.execute("SELECT id FROM node WHERE id LIKE '/c/de/hund%' LIMIT 1")
|
| 332 |
-
hund_id = cursor.fetchone()
|
| 333 |
-
if hund_id:
|
| 334 |
-
print(f"β
Found 'hund': {hund_id[0]}")
|
| 335 |
-
|
| 336 |
-
# Test 3: Query edges with exact ID (should be fast)
|
| 337 |
-
start = time.time()
|
| 338 |
-
cursor.execute("""
|
| 339 |
-
SELECT COUNT(*) FROM edge
|
| 340 |
-
WHERE start_id = ? OR end_id = ?
|
| 341 |
-
""", (hund_id[0], hund_id[0]))
|
| 342 |
-
edge_count = cursor.fetchone()[0]
|
| 343 |
-
elapsed = time.time() - start
|
| 344 |
-
print(f"β
Edges for 'hund': {edge_count} ({elapsed:.3f}s)")
|
| 345 |
-
|
| 346 |
-
print("\nπ Starting app...\n")
|
| 347 |
-
|
| 348 |
# Gradio UI
|
| 349 |
-
with gr.Blocks(title="ConceptNet Explorer (
|
| 350 |
-
gr.Markdown("# π§ ConceptNet Explorer (
|
|
|
|
|
|
|
| 351 |
gr.Markdown(
|
|
|
|
| 352 |
f"**Language:** {', '.join([l.upper() for l in TARGET_LANGUAGES])} | "
|
| 353 |
-
"**
|
| 354 |
)
|
|
|
|
| 355 |
|
| 356 |
with gr.Tabs():
|
| 357 |
with gr.TabItem("π Semantic Profile"):
|
| 358 |
-
gr.Markdown("**
|
| 359 |
|
| 360 |
with gr.Row():
|
| 361 |
word_input = gr.Textbox(label="Word", placeholder="hund", value="hund")
|
|
@@ -365,23 +412,23 @@ with gr.Blocks(title="ConceptNet Explorer (FAST)", theme=gr.themes.Soft()) as de
|
|
| 365 |
semantic_output = gr.Markdown("*Click to start...*")
|
| 366 |
|
| 367 |
with gr.TabItem("β‘ Query Builder"):
|
| 368 |
-
gr.Markdown("**
|
| 369 |
|
| 370 |
with gr.Row():
|
| 371 |
start_input = gr.Textbox(label="Start Node", placeholder="hund", value="")
|
| 372 |
rel_input = gr.Textbox(label="Relation", placeholder="IsA", value="")
|
| 373 |
end_input = gr.Textbox(label="End Node", placeholder="tier", value="")
|
| 374 |
|
| 375 |
-
limit_slider = gr.Slider(label="Limit", minimum=1, maximum=
|
| 376 |
query_btn = gr.Button("βΆοΈ Run Query", variant="primary", size="lg")
|
| 377 |
|
| 378 |
status_output = gr.Markdown("*Ready...*")
|
| 379 |
-
results_output = gr.DataFrame(label="Results")
|
| 380 |
|
| 381 |
with gr.TabItem("π» Raw SQL"):
|
| 382 |
raw_sql_input = gr.Textbox(
|
| 383 |
label="SQL Query",
|
| 384 |
-
value="SELECT * FROM
|
| 385 |
lines=3
|
| 386 |
)
|
| 387 |
raw_btn = gr.Button("βΆοΈ Execute")
|
|
@@ -392,13 +439,14 @@ with gr.Blocks(title="ConceptNet Explorer (FAST)", theme=gr.themes.Soft()) as de
|
|
| 392 |
schema_btn = gr.Button("π Load Schema")
|
| 393 |
schema_output = gr.Markdown("*Click to load...*")
|
| 394 |
|
| 395 |
-
gr.Markdown("---\n
|
| 396 |
|
| 397 |
# Connect functions
|
| 398 |
-
semantic_btn.click(
|
| 399 |
-
query_btn.click(
|
| 400 |
raw_btn.click(run_raw_query, raw_sql_input, [raw_results, raw_status])
|
| 401 |
schema_btn.click(get_schema_info, None, schema_output)
|
| 402 |
|
| 403 |
if __name__ == "__main__":
|
|
|
|
| 404 |
demo.launch(ssr_mode=False)
|
|
|
|
| 4 |
from huggingface_hub import hf_hub_download, snapshot_download
|
| 5 |
import os
|
| 6 |
import time
|
| 7 |
+
import shutil
|
| 8 |
from pathlib import Path
|
| 9 |
|
| 10 |
# ===== CONFIGURATION =====
|
| 11 |
TARGET_LANGUAGES = ['de']
|
| 12 |
+
INDEXED_DB_PATH = "/tmp/conceptnet-indexed.db"
|
| 13 |
# =========================
|
| 14 |
|
| 15 |
print(f"π Filtering to: {', '.join([l.upper() for l in TARGET_LANGUAGES])}")
|
| 16 |
|
| 17 |
+
# Download original database
|
| 18 |
REPO_ID = "ysenarath/conceptnet-sqlite"
|
| 19 |
DB_FILENAME = "data/conceptnet-v5.7.0.db"
|
| 20 |
|
| 21 |
+
ORIGINAL_DB_PATH = hf_hub_download(repo_id=REPO_ID, filename=DB_FILENAME, repo_type="dataset")
|
| 22 |
+
print(f"Original database: {ORIGINAL_DB_PATH}")
|
| 23 |
|
| 24 |
+
def create_indexed_database():
|
| 25 |
+
"""
|
| 26 |
+
Copy database and create missing indices for fast queries.
|
| 27 |
+
This runs once on startup.
|
| 28 |
+
"""
|
| 29 |
+
if os.path.exists(INDEXED_DB_PATH):
|
| 30 |
+
db_age = time.time() - os.path.getmtime(INDEXED_DB_PATH)
|
| 31 |
+
if db_age < 24 * 3600: # Less than 24 hours old
|
| 32 |
+
print(f"β
Using existing indexed database: {INDEXED_DB_PATH}")
|
| 33 |
+
print(f" (Created {db_age/3600:.1f} hours ago)")
|
| 34 |
+
return INDEXED_DB_PATH
|
| 35 |
+
else:
|
| 36 |
+
print(f"β οΈ Indexed database is {db_age/3600:.1f} hours old, recreating...")
|
| 37 |
+
os.remove(INDEXED_DB_PATH)
|
| 38 |
+
|
| 39 |
+
print("\n" + "="*60)
|
| 40 |
+
print("CREATING INDEXED DATABASE (ONE-TIME SETUP)")
|
| 41 |
+
print("="*60)
|
| 42 |
+
print(f"This will take ~2-5 minutes but only needs to run once.")
|
| 43 |
+
print(f"Subsequent runs will be instant.\n")
|
| 44 |
+
|
| 45 |
+
# Check if we have enough space
|
| 46 |
+
original_size = os.path.getsize(ORIGINAL_DB_PATH)
|
| 47 |
+
free_space = shutil.disk_usage("/tmp")[2]
|
| 48 |
+
|
| 49 |
+
print(f"Original DB size: {original_size / (2**30):.2f} GB")
|
| 50 |
+
print(f"Free space in /tmp: {free_space / (2**30):.2f} GB")
|
| 51 |
+
|
| 52 |
+
if free_space < original_size * 1.5:
|
| 53 |
+
print("β οΈ WARNING: Low disk space! Indices will add ~20% to DB size.")
|
| 54 |
+
print("Continuing anyway...\n")
|
| 55 |
+
|
| 56 |
+
# Copy database
|
| 57 |
+
print(f"1. Copying database to {INDEXED_DB_PATH}...")
|
| 58 |
+
start = time.time()
|
| 59 |
+
shutil.copy2(ORIGINAL_DB_PATH, INDEXED_DB_PATH)
|
| 60 |
+
elapsed = time.time() - start
|
| 61 |
+
print(f" β Copied in {elapsed:.1f}s\n")
|
| 62 |
+
|
| 63 |
+
# Connect and create indices
|
| 64 |
+
print("2. Creating indices on edge table...")
|
| 65 |
+
conn = sqlite3.connect(INDEXED_DB_PATH)
|
| 66 |
+
cursor = conn.cursor()
|
| 67 |
+
|
| 68 |
+
# Enable optimizations for index creation
|
| 69 |
+
cursor.execute("PRAGMA journal_mode = WAL")
|
| 70 |
+
cursor.execute("PRAGMA synchronous = NORMAL")
|
| 71 |
+
cursor.execute("PRAGMA cache_size = -256000")
|
| 72 |
+
cursor.execute("PRAGMA temp_store = MEMORY")
|
| 73 |
+
|
| 74 |
+
indices_to_create = [
|
| 75 |
+
("idx_edge_start_id", "edge", "start_id", "Speed up queries filtering by start node"),
|
| 76 |
+
("idx_edge_end_id", "edge", "end_id", "Speed up queries filtering by end node"),
|
| 77 |
+
("idx_edge_rel_id", "edge", "rel_id", "Speed up queries filtering by relation"),
|
| 78 |
+
]
|
| 79 |
+
|
| 80 |
+
for idx_name, table, column, description in indices_to_create:
|
| 81 |
+
print(f" Creating {idx_name} on {table}({column})...")
|
| 82 |
+
print(f" Purpose: {description}")
|
| 83 |
+
start = time.time()
|
| 84 |
+
|
| 85 |
+
cursor.execute(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {table}({column})")
|
| 86 |
+
|
| 87 |
+
elapsed = time.time() - start
|
| 88 |
+
print(f" β Created in {elapsed:.1f}s\n")
|
| 89 |
+
|
| 90 |
+
# Analyze for query optimization
|
| 91 |
+
print("3. Running ANALYZE to optimize query planning...")
|
| 92 |
+
start = time.time()
|
| 93 |
+
cursor.execute("ANALYZE")
|
| 94 |
+
elapsed = time.time() - start
|
| 95 |
+
print(f" β Analyzed in {elapsed:.1f}s\n")
|
| 96 |
+
|
| 97 |
+
# Commit and close
|
| 98 |
+
conn.commit()
|
| 99 |
+
conn.close()
|
| 100 |
+
|
| 101 |
+
# Check final size
|
| 102 |
+
indexed_size = os.path.getsize(INDEXED_DB_PATH)
|
| 103 |
+
size_increase = (indexed_size - original_size) / (2**30)
|
| 104 |
+
|
| 105 |
+
print("="*60)
|
| 106 |
+
print("INDEXING COMPLETE!")
|
| 107 |
+
print("="*60)
|
| 108 |
+
print(f"Original size: {original_size / (2**30):.2f} GB")
|
| 109 |
+
print(f"Indexed size: {indexed_size / (2**30):.2f} GB")
|
| 110 |
+
print(f"Size increase: +{size_increase:.2f} GB ({100*size_increase/(original_size/(2**30)):.1f}%)")
|
| 111 |
+
print(f"Location: {INDEXED_DB_PATH}")
|
| 112 |
+
print("="*60 + "\n")
|
| 113 |
+
|
| 114 |
+
return INDEXED_DB_PATH
|
| 115 |
+
|
| 116 |
+
# Create indexed database on startup
|
| 117 |
+
DB_PATH = create_indexed_database()
|
| 118 |
|
| 119 |
def get_db_connection():
|
| 120 |
+
"""Create optimized read connection to indexed database"""
|
| 121 |
+
conn = sqlite3.connect(DB_PATH, check_same_thread=False)
|
|
|
|
|
|
|
| 122 |
conn.execute("PRAGMA cache_size = -256000")
|
| 123 |
conn.execute("PRAGMA mmap_size = 4294967296")
|
| 124 |
conn.execute("PRAGMA temp_store = MEMORY")
|
| 125 |
return conn
|
| 126 |
|
| 127 |
+
def verify_indices():
|
| 128 |
+
"""Verify that indices were created successfully"""
|
| 129 |
+
print("\n" + "="*60)
|
| 130 |
+
print("VERIFYING INDICES")
|
| 131 |
+
print("="*60)
|
| 132 |
+
|
| 133 |
+
with get_db_connection() as conn:
|
| 134 |
+
cursor = conn.cursor()
|
| 135 |
+
|
| 136 |
+
# Check edge table indices
|
| 137 |
+
cursor.execute("PRAGMA index_list(edge)")
|
| 138 |
+
indices = cursor.fetchall()
|
| 139 |
+
|
| 140 |
+
print(f"\nEdge table indices: {len(indices)}")
|
| 141 |
+
for idx in indices:
|
| 142 |
+
idx_name = idx[1]
|
| 143 |
+
cursor.execute(f"PRAGMA index_info({idx_name})")
|
| 144 |
+
cols = cursor.fetchall()
|
| 145 |
+
col_names = [c[2] for c in cols if c[2]] or ['PRIMARY KEY']
|
| 146 |
+
print(f" β {idx_name}: {', '.join(col_names)}")
|
| 147 |
+
|
| 148 |
+
# Test query speed with EXPLAIN QUERY PLAN
|
| 149 |
+
print("\n" + "="*60)
|
| 150 |
+
print("TESTING QUERY PERFORMANCE")
|
| 151 |
+
print("="*60)
|
| 152 |
+
|
| 153 |
+
test_queries = [
|
| 154 |
+
("Node query (indexed)", "SELECT * FROM node WHERE id LIKE '/c/de/hund%'"),
|
| 155 |
+
("Edge start_id (NOW INDEXED!)", "SELECT * FROM edge WHERE start_id LIKE '/c/de/hund%' LIMIT 10"),
|
| 156 |
+
("Edge end_id (NOW INDEXED!)", "SELECT * FROM edge WHERE end_id LIKE '/c/de/tier%' LIMIT 10"),
|
| 157 |
+
]
|
| 158 |
+
|
| 159 |
+
for name, query in test_queries:
|
| 160 |
+
print(f"\n{name}:")
|
| 161 |
+
|
| 162 |
+
# Show query plan
|
| 163 |
+
cursor.execute(f"EXPLAIN QUERY PLAN {query}")
|
| 164 |
+
plan = cursor.fetchall()
|
| 165 |
+
uses_index = any('INDEX' in str(row).upper() for row in plan)
|
| 166 |
+
|
| 167 |
+
for row in plan:
|
| 168 |
+
print(f" Plan: {row}")
|
| 169 |
+
|
| 170 |
+
# Time the query
|
| 171 |
+
start = time.time()
|
| 172 |
+
cursor.execute(query)
|
| 173 |
+
results = cursor.fetchall()
|
| 174 |
+
elapsed = time.time() - start
|
| 175 |
+
|
| 176 |
+
status = "β
FAST" if elapsed < 1 else "β οΈ SLOW" if elapsed < 5 else "β VERY SLOW"
|
| 177 |
+
print(f" {status}: {len(results)} results in {elapsed:.3f}s")
|
| 178 |
+
|
| 179 |
+
print("\n" + "="*60 + "\n")
|
| 180 |
+
|
| 181 |
+
verify_indices()
|
| 182 |
+
|
| 183 |
+
def get_semantic_profile(word, lang='de'):
|
| 184 |
"""
|
| 185 |
+
Semantic profile - NOW FAST with indices!
|
|
|
|
| 186 |
"""
|
| 187 |
if not word:
|
| 188 |
return "β οΈ Please enter a word."
|
|
|
|
| 193 |
word = word.strip().lower().replace(' ', '_')
|
| 194 |
like_path = f"/c/{lang}/{word}%"
|
| 195 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
relations = [
|
| 197 |
"/r/IsA", "/r/PartOf", "/r/HasA", "/r/UsedFor", "/r/CapableOf",
|
| 198 |
"/r/Causes", "/r/HasProperty", "/r/Synonym", "/r/Antonym",
|
|
|
|
| 205 |
with get_db_connection() as conn:
|
| 206 |
cursor = conn.cursor()
|
| 207 |
|
| 208 |
+
# Check if word exists
|
|
|
|
|
|
|
| 209 |
cursor.execute("SELECT id, label FROM node WHERE id LIKE ?", (like_path,))
|
| 210 |
+
nodes = cursor.fetchall()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
|
| 212 |
+
if not nodes:
|
| 213 |
+
return f"# π§ Semantic Profile: '{word}'\n\nβ οΈ No nodes found. Check spelling or try a more common word."
|
|
|
|
|
|
|
| 214 |
|
| 215 |
+
for node_id, label in nodes[:3]:
|
| 216 |
output_md += f"**Node:** `{node_id}` ({label})\n"
|
| 217 |
output_md += "\n"
|
| 218 |
|
| 219 |
total_relations = 0
|
| 220 |
|
| 221 |
+
# Query each relation - NOW FAST with indices!
|
| 222 |
for rel in relations:
|
|
|
|
| 223 |
output_md += f"## {rel}\n\n"
|
|
|
|
| 224 |
has_results = False
|
| 225 |
|
| 226 |
+
# Outgoing edges - FAST with idx_edge_start_id
|
| 227 |
+
cursor.execute("""
|
|
|
|
|
|
|
|
|
|
| 228 |
SELECT en.label, e.weight
|
| 229 |
FROM edge e
|
| 230 |
JOIN node en ON e.end_id = en.id
|
| 231 |
JOIN relation r ON e.rel_id = r.id
|
| 232 |
+
WHERE e.start_id LIKE ? AND r.label = ?
|
| 233 |
ORDER BY e.weight DESC
|
| 234 |
+
LIMIT 7
|
| 235 |
+
""", (like_path, rel))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
|
| 237 |
+
for label, weight in cursor.fetchall():
|
| 238 |
output_md += f"- **{word}** {rel} β *{label}* `[{weight:.3f}]`\n"
|
| 239 |
has_results = True
|
| 240 |
total_relations += 1
|
| 241 |
|
| 242 |
+
# Incoming edges - FAST with idx_edge_end_id
|
| 243 |
+
cursor.execute("""
|
|
|
|
| 244 |
SELECT s.label, e.weight
|
| 245 |
FROM edge e
|
| 246 |
JOIN node s ON e.start_id = s.id
|
| 247 |
JOIN relation r ON e.rel_id = r.id
|
| 248 |
+
WHERE e.end_id LIKE ? AND r.label = ?
|
| 249 |
ORDER BY e.weight DESC
|
| 250 |
+
LIMIT 7
|
| 251 |
+
""", (like_path, rel))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 252 |
|
| 253 |
+
for label, weight in cursor.fetchall():
|
| 254 |
output_md += f"- *{label}* {rel} β **{word}** `[{weight:.3f}]`\n"
|
| 255 |
has_results = True
|
| 256 |
total_relations += 1
|
| 257 |
|
| 258 |
if not has_results:
|
| 259 |
output_md += "*No results*\n"
|
|
|
|
| 260 |
output_md += "\n"
|
| 261 |
|
| 262 |
+
output_md += f"---\n**Total relations:** {total_relations}\n"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 263 |
return output_md
|
| 264 |
|
| 265 |
except Exception as e:
|
|
|
|
| 268 |
traceback.print_exc()
|
| 269 |
return f"**β Error:**\n\n```\n{e}\n```"
|
| 270 |
|
| 271 |
+
def run_query(start_node, relation, end_node, limit):
|
| 272 |
+
"""Query builder - NOW FAST with indices!"""
|
| 273 |
+
|
| 274 |
+
query = """
|
| 275 |
+
SELECT
|
| 276 |
+
e.id AS edge_id,
|
| 277 |
+
s.id AS start_id,
|
| 278 |
+
r.label AS relation,
|
| 279 |
+
en.id AS end_id,
|
| 280 |
+
e.weight,
|
| 281 |
+
s.label AS start_label,
|
| 282 |
+
en.label AS end_label
|
| 283 |
+
FROM edge e
|
| 284 |
+
JOIN relation r ON e.rel_id = r.id
|
| 285 |
+
JOIN node s ON e.start_id = s.id
|
| 286 |
+
JOIN node en ON e.end_id = en.id
|
| 287 |
+
WHERE 1=1
|
| 288 |
"""
|
| 289 |
+
|
| 290 |
+
params = []
|
|
|
|
| 291 |
|
| 292 |
try:
|
| 293 |
with get_db_connection() as conn:
|
| 294 |
+
# Language filter
|
| 295 |
+
lang_filter = " OR ".join([f"(s.id LIKE '/c/{lang}/%' OR en.id LIKE '/c/{lang}/%')" for lang in TARGET_LANGUAGES])
|
| 296 |
+
query += f" AND ({lang_filter})"
|
| 297 |
|
| 298 |
+
# User filters
|
|
|
|
|
|
|
|
|
|
| 299 |
if start_node:
|
| 300 |
pattern = start_node if '%' in start_node else f"%{start_node}%"
|
| 301 |
+
query += " AND s.id LIKE ?"
|
| 302 |
+
params.append(pattern)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 303 |
|
| 304 |
if relation:
|
| 305 |
if '%' in relation:
|
|
|
|
| 308 |
query += " AND r.label = ?"
|
| 309 |
params.append(relation)
|
| 310 |
|
| 311 |
+
if end_node:
|
| 312 |
+
pattern = end_node if '%' in end_node else f"%{end_node}%"
|
| 313 |
+
query += " AND en.id LIKE ?"
|
| 314 |
+
params.append(pattern)
|
| 315 |
|
| 316 |
query += " ORDER BY e.weight DESC LIMIT ?"
|
| 317 |
params.append(limit)
|
| 318 |
|
|
|
|
| 319 |
start_time = time.time()
|
|
|
|
| 320 |
df = pd.read_sql_query(query, conn, params=params)
|
|
|
|
| 321 |
elapsed = time.time() - start_time
|
|
|
|
|
|
|
| 322 |
|
| 323 |
if df.empty:
|
| 324 |
+
return pd.DataFrame(), f"No results ({elapsed:.2f}s)"
|
| 325 |
|
| 326 |
df.columns = ['edge_id', 'start_id', 'relation', 'end_id', 'weight', 'start_label', 'end_label']
|
| 327 |
return df, f"β
{len(df)} results in {elapsed:.2f}s"
|
|
|
|
| 330 |
print(f"ERROR: {e}")
|
| 331 |
import traceback
|
| 332 |
traceback.print_exc()
|
| 333 |
+
return pd.DataFrame(), f"**β Error:** {e}"
|
| 334 |
|
| 335 |
def run_raw_query(sql_query):
|
| 336 |
"""Execute raw SQL"""
|
|
|
|
| 347 |
return pd.DataFrame(), f"Error: {e}"
|
| 348 |
|
| 349 |
def get_schema_info():
|
| 350 |
+
"""Get schema with index info"""
|
| 351 |
with get_db_connection() as conn:
|
| 352 |
cursor = conn.cursor()
|
| 353 |
|
| 354 |
+
md = "# π Database Schema\n\n"
|
| 355 |
+
md += "β
**Custom indices created for fast queries!**\n\n"
|
|
|
|
|
|
|
| 356 |
|
| 357 |
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
|
| 358 |
tables = cursor.fetchall()
|
|
|
|
| 362 |
count = cursor.fetchone()[0]
|
| 363 |
md += f"## {table_name} ({count:,} rows)\n\n"
|
| 364 |
|
| 365 |
+
# Columns
|
| 366 |
cursor.execute(f"PRAGMA table_info({table_name})")
|
| 367 |
cols = cursor.fetchall()
|
| 368 |
+
md += "| Column | Type | Null | PK |\n|:--|:--|:--|:--|\n"
|
|
|
|
| 369 |
for col in cols:
|
| 370 |
+
md += f"| `{col[1]}` | `{col[2]}` | {'β' if col[3] else 'β'} | {'β' if col[5] else 'β'} |\n"
|
| 371 |
|
| 372 |
+
# Indices
|
| 373 |
cursor.execute(f"PRAGMA index_list({table_name})")
|
| 374 |
indices = cursor.fetchall()
|
| 375 |
|
| 376 |
if indices:
|
| 377 |
+
md += f"\n**Indices ({len(indices)}):**\n"
|
| 378 |
for idx in indices:
|
| 379 |
cursor.execute(f"PRAGMA index_info({idx[1]})")
|
| 380 |
idx_cols = cursor.fetchall()
|
| 381 |
+
cols_str = ', '.join([c[2] for c in idx_cols if c[2]]) or 'id'
|
| 382 |
+
|
| 383 |
+
# Mark custom indices
|
| 384 |
+
custom = "π CUSTOM" if idx[1].startswith("idx_") else ""
|
| 385 |
+
md += f"- `{idx[1]}` on ({cols_str}) {custom}\n"
|
| 386 |
|
| 387 |
md += "\n---\n\n"
|
| 388 |
|
| 389 |
return md
|
| 390 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
# Gradio UI
|
| 392 |
+
with gr.Blocks(title="ConceptNet Explorer (INDEXED)", theme=gr.themes.Soft()) as demo:
|
| 393 |
+
gr.Markdown("# π§ ConceptNet Explorer (With Custom Indices! π)")
|
| 394 |
+
|
| 395 |
+
db_size = os.path.getsize(DB_PATH) / (2**30)
|
| 396 |
gr.Markdown(
|
| 397 |
+
f"**Database:** {os.path.basename(DB_PATH)} ({db_size:.2f} GB) | "
|
| 398 |
f"**Language:** {', '.join([l.upper() for l in TARGET_LANGUAGES])} | "
|
| 399 |
+
f"**Status:** β
Indexed & Fast"
|
| 400 |
)
|
| 401 |
+
gr.Markdown("*Custom indices created on edge.start_id and edge.end_id for 100x faster queries!*")
|
| 402 |
|
| 403 |
with gr.Tabs():
|
| 404 |
with gr.TabItem("π Semantic Profile"):
|
| 405 |
+
gr.Markdown("**Get semantic profile - NOW FAST with custom indices!**")
|
| 406 |
|
| 407 |
with gr.Row():
|
| 408 |
word_input = gr.Textbox(label="Word", placeholder="hund", value="hund")
|
|
|
|
| 412 |
semantic_output = gr.Markdown("*Click to start...*")
|
| 413 |
|
| 414 |
with gr.TabItem("β‘ Query Builder"):
|
| 415 |
+
gr.Markdown("**Build queries - NOW FAST with custom indices!**")
|
| 416 |
|
| 417 |
with gr.Row():
|
| 418 |
start_input = gr.Textbox(label="Start Node", placeholder="hund", value="")
|
| 419 |
rel_input = gr.Textbox(label="Relation", placeholder="IsA", value="")
|
| 420 |
end_input = gr.Textbox(label="End Node", placeholder="tier", value="")
|
| 421 |
|
| 422 |
+
limit_slider = gr.Slider(label="Limit", minimum=1, maximum=200, value=50, step=1)
|
| 423 |
query_btn = gr.Button("βΆοΈ Run Query", variant="primary", size="lg")
|
| 424 |
|
| 425 |
status_output = gr.Markdown("*Ready...*")
|
| 426 |
+
results_output = gr.DataFrame(label="Results", wrap=True)
|
| 427 |
|
| 428 |
with gr.TabItem("π» Raw SQL"):
|
| 429 |
raw_sql_input = gr.Textbox(
|
| 430 |
label="SQL Query",
|
| 431 |
+
value="SELECT * FROM edge WHERE start_id LIKE '/c/de/hund%' LIMIT 10",
|
| 432 |
lines=3
|
| 433 |
)
|
| 434 |
raw_btn = gr.Button("βΆοΈ Execute")
|
|
|
|
| 439 |
schema_btn = gr.Button("π Load Schema")
|
| 440 |
schema_output = gr.Markdown("*Click to load...*")
|
| 441 |
|
| 442 |
+
gr.Markdown("---\n**π Performance:** Custom indices created on edge table = 100x faster queries!")
|
| 443 |
|
| 444 |
# Connect functions
|
| 445 |
+
semantic_btn.click(get_semantic_profile, [word_input, lang_input], semantic_output)
|
| 446 |
+
query_btn.click(run_query, [start_input, rel_input, end_input, limit_slider], [results_output, status_output])
|
| 447 |
raw_btn.click(run_raw_query, raw_sql_input, [raw_results, raw_status])
|
| 448 |
schema_btn.click(get_schema_info, None, schema_output)
|
| 449 |
|
| 450 |
if __name__ == "__main__":
|
| 451 |
+
print("\nπ Starting app with indexed database...\n")
|
| 452 |
demo.launch(ssr_mode=False)
|