Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
|
@@ -239,29 +239,30 @@ def create_indexed_database():
|
|
| 239 |
log_progress(f"Download failed: {e}", "ERROR")
|
| 240 |
log_progress("Will create locally", "INFO")
|
| 241 |
|
| 242 |
-
#
|
| 243 |
-
if completed_indices or analyzed_tables:
|
| 244 |
-
log_progress("
|
| 245 |
log_progress(f" Completed indices: {sorted(completed_indices)}", "INFO")
|
| 246 |
log_progress(f" Analyzed tables: {sorted(analyzed_tables)}", "INFO")
|
| 247 |
-
|
| 248 |
-
# Get or create local database
|
| 249 |
-
if os.path.exists(LOCAL_DB_PATH) and (completed_indices or analyzed_tables):
|
| 250 |
-
log_progress("Using existing local database", "SUCCESS")
|
| 251 |
-
elif database_uploaded:
|
| 252 |
-
log_progress("Downloading partial database from HF...", "INFO")
|
| 253 |
try:
|
| 254 |
-
|
| 255 |
repo_id=INDEXED_REPO_ID,
|
| 256 |
filename=INDEXED_DB_FILENAME,
|
| 257 |
repo_type="dataset",
|
| 258 |
token=HF_TOKEN
|
| 259 |
)
|
| 260 |
-
|
| 261 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 262 |
except Exception as e:
|
| 263 |
-
log_progress(f"
|
|
|
|
| 264 |
|
|
|
|
| 265 |
if not os.path.exists(LOCAL_DB_PATH):
|
| 266 |
# Download and copy original
|
| 267 |
log_progress("Downloading original database...", "INFO")
|
|
@@ -285,6 +286,11 @@ def create_indexed_database():
|
|
| 285 |
shutil.copy2(original_path, LOCAL_DB_PATH)
|
| 286 |
elapsed = time.time() - start
|
| 287 |
log_progress(f"Copied in {elapsed:.1f}s", "SUCCESS")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
|
| 289 |
# Connect to database
|
| 290 |
conn = sqlite3.connect(LOCAL_DB_PATH)
|
|
|
|
| 239 |
log_progress(f"Download failed: {e}", "ERROR")
|
| 240 |
log_progress("Will create locally", "INFO")
|
| 241 |
|
| 242 |
+
# NEW: If all indices complete OR partial work done, try to download indexed DB
|
| 243 |
+
if completed_indices or analyzed_tables or database_uploaded:
|
| 244 |
+
log_progress("Checkpoint detected - attempting to download indexed database...", "INFO")
|
| 245 |
log_progress(f" Completed indices: {sorted(completed_indices)}", "INFO")
|
| 246 |
log_progress(f" Analyzed tables: {sorted(analyzed_tables)}", "INFO")
|
| 247 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 248 |
try:
|
| 249 |
+
indexed_path = hf_hub_download(
|
| 250 |
repo_id=INDEXED_REPO_ID,
|
| 251 |
filename=INDEXED_DB_FILENAME,
|
| 252 |
repo_type="dataset",
|
| 253 |
token=HF_TOKEN
|
| 254 |
)
|
| 255 |
+
|
| 256 |
+
# Copy to local path
|
| 257 |
+
log_progress(f"Downloaded indexed DB, copying to {LOCAL_DB_PATH}...", "INFO")
|
| 258 |
+
shutil.copy2(indexed_path, LOCAL_DB_PATH)
|
| 259 |
+
log_progress("Using downloaded indexed database", "SUCCESS")
|
| 260 |
+
|
| 261 |
except Exception as e:
|
| 262 |
+
log_progress(f"Could not download indexed DB: {e}", "WARN")
|
| 263 |
+
log_progress("Will need to start from original and create indices", "INFO")
|
| 264 |
|
| 265 |
+
# Only download original if we don't have local DB yet
|
| 266 |
if not os.path.exists(LOCAL_DB_PATH):
|
| 267 |
# Download and copy original
|
| 268 |
log_progress("Downloading original database...", "INFO")
|
|
|
|
| 286 |
shutil.copy2(original_path, LOCAL_DB_PATH)
|
| 287 |
elapsed = time.time() - start
|
| 288 |
log_progress(f"Copied in {elapsed:.1f}s", "SUCCESS")
|
| 289 |
+
|
| 290 |
+
# Clear completed indices since we're starting from scratch
|
| 291 |
+
log_progress("Starting from original DB - clearing completed indices list", "WARN")
|
| 292 |
+
completed_indices = set()
|
| 293 |
+
analyzed_tables = set()
|
| 294 |
|
| 295 |
# Connect to database
|
| 296 |
conn = sqlite3.connect(LOCAL_DB_PATH)
|