cstr commited on
Commit
228bc82
·
verified ·
1 Parent(s): 33e2835

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +19 -13
app.py CHANGED
@@ -239,29 +239,30 @@ def create_indexed_database():
239
  log_progress(f"Download failed: {e}", "ERROR")
240
  log_progress("Will create locally", "INFO")
241
 
242
- # Check for partial progress
243
- if completed_indices or analyzed_tables:
244
- log_progress("Resuming from checkpoint:", "INFO")
245
  log_progress(f" Completed indices: {sorted(completed_indices)}", "INFO")
246
  log_progress(f" Analyzed tables: {sorted(analyzed_tables)}", "INFO")
247
-
248
- # Get or create local database
249
- if os.path.exists(LOCAL_DB_PATH) and (completed_indices or analyzed_tables):
250
- log_progress("Using existing local database", "SUCCESS")
251
- elif database_uploaded:
252
- log_progress("Downloading partial database from HF...", "INFO")
253
  try:
254
- remote_db = hf_hub_download(
255
  repo_id=INDEXED_REPO_ID,
256
  filename=INDEXED_DB_FILENAME,
257
  repo_type="dataset",
258
  token=HF_TOKEN
259
  )
260
- shutil.copy2(remote_db, LOCAL_DB_PATH)
261
- log_progress("Downloaded partial database", "SUCCESS")
 
 
 
 
262
  except Exception as e:
263
- log_progress(f"No partial database: {e}", "INFO")
 
264
 
 
265
  if not os.path.exists(LOCAL_DB_PATH):
266
  # Download and copy original
267
  log_progress("Downloading original database...", "INFO")
@@ -285,6 +286,11 @@ def create_indexed_database():
285
  shutil.copy2(original_path, LOCAL_DB_PATH)
286
  elapsed = time.time() - start
287
  log_progress(f"Copied in {elapsed:.1f}s", "SUCCESS")
 
 
 
 
 
288
 
289
  # Connect to database
290
  conn = sqlite3.connect(LOCAL_DB_PATH)
 
239
  log_progress(f"Download failed: {e}", "ERROR")
240
  log_progress("Will create locally", "INFO")
241
 
242
+ # NEW: If all indices complete OR partial work done, try to download indexed DB
243
+ if completed_indices or analyzed_tables or database_uploaded:
244
+ log_progress("Checkpoint detected - attempting to download indexed database...", "INFO")
245
  log_progress(f" Completed indices: {sorted(completed_indices)}", "INFO")
246
  log_progress(f" Analyzed tables: {sorted(analyzed_tables)}", "INFO")
247
+
 
 
 
 
 
248
  try:
249
+ indexed_path = hf_hub_download(
250
  repo_id=INDEXED_REPO_ID,
251
  filename=INDEXED_DB_FILENAME,
252
  repo_type="dataset",
253
  token=HF_TOKEN
254
  )
255
+
256
+ # Copy to local path
257
+ log_progress(f"Downloaded indexed DB, copying to {LOCAL_DB_PATH}...", "INFO")
258
+ shutil.copy2(indexed_path, LOCAL_DB_PATH)
259
+ log_progress("Using downloaded indexed database", "SUCCESS")
260
+
261
  except Exception as e:
262
+ log_progress(f"Could not download indexed DB: {e}", "WARN")
263
+ log_progress("Will need to start from original and create indices", "INFO")
264
 
265
+ # Only download original if we don't have local DB yet
266
  if not os.path.exists(LOCAL_DB_PATH):
267
  # Download and copy original
268
  log_progress("Downloading original database...", "INFO")
 
286
  shutil.copy2(original_path, LOCAL_DB_PATH)
287
  elapsed = time.time() - start
288
  log_progress(f"Copied in {elapsed:.1f}s", "SUCCESS")
289
+
290
+ # Clear completed indices since we're starting from scratch
291
+ log_progress("Starting from original DB - clearing completed indices list", "WARN")
292
+ completed_indices = set()
293
+ analyzed_tables = set()
294
 
295
  # Connect to database
296
  conn = sqlite3.connect(LOCAL_DB_PATH)