Update app.py

app.py

@@ -13,6 +13,15 @@ import torch
 from torch.utils.data import Dataset, DataLoader
 from sklearn.model_selection import train_test_split
 import numpy as np
+import nest_asyncio # ⭐️ FIX 2: Added nest_asyncio for stability
+import sys
+
+# --- SYSTEM FIXES ---
+# ⭐️ FIX 2: Apply nest_asyncio to prevent EventLoop/Gradio conflicts (Invalid file descriptor: -1)
+try:
+    nest_asyncio.apply()
+except Exception as e:
+    print(f"Warning: Could not apply nest_asyncio: {e}")
 
 # --- CONFIGURATION ---
 DATASET_DIR = "dataset_ml_final_v2"
@@ -26,20 +35,17 @@ MODEL = None
 DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
 try:
-    # ---------------------------------------------------------
-    # FIX APPLIED HERE:
-    # 1. Removed AdamW from transformers import
-    # 2. Added AdamW from torch.optim
-    # ---------------------------------------------------------
-    from transformers import AutoTokenizer, AutoModelForSequenceClassification, get_linear_schedule_with_warmup
+    from transformers import AutoTokenizer, AutoModelForSequenceClassification, get_linear_schedule_with_warmup, logging
     from torch.optim import AdamW
     
+    # Suppress heavy warnings from transformers
+    logging.set_verbosity_error()
+    
     print("Attempting to load Longformer Tokenizer...")
     TOKENIZER = AutoTokenizer.from_pretrained("allenai/longformer-base-4096")
     print("✅ Tokenizer loaded successfully.")
 except Exception as e:
     print(f"⚠️ Tokenizer loading error: {e}")
-    # Fallback for debugging if transformers fails entirely
     AdamW = None 
 
 # --- ERAS (10 Distinct Periods) ---
@@ -154,7 +160,7 @@ def download_book(identifier, title, year, era_label, min_char_limit=5000):
     content = ""
     for url in urls:
         try:
-            r = requests.get(url, timeout=10)
+            r = requests.get(url, timeout=15) # Increased timeout for robustness
             if r.status_code == 200:
                 content = r.text
                 break
@@ -241,16 +247,25 @@ def generate_dataset(total_books_needed, progress=gr.Progress()):
                 print(f"  > Standard Search #{attempts}: {topic}")
 
             try:
-                search = internetarchive.search_items(
+                search_generator = internetarchive.search_items(
                     query, 
                     sorts=['downloads desc'], 
                     fields=['identifier', 'title', 'date', 'year']
                 )
                 
-                results_found = 0
-                for res in search:
+                # ⭐️ FIX 1: Pre-fetch a batch of results to close the search connection quickly
+                search_results_batch = []
+                # Check a reasonable number of items before going back to the search
+                max_check_per_query = (50 if era_label == "1_Late_Medieval" else (30 if is_hard_era else 10))
+                for i, item in enumerate(search_generator):
+                    search_results_batch.append(item)
+                    if i >= max_check_per_query: break
+                
+                results_found = len(search_results_batch)
+                
+                # Now iterate through the SAFE pre-fetched list
+                for res in search_results_batch:
                     if collected >= books_per_era: break
-                    results_found += 1
                     
                     id_ = res.get('identifier')
                     raw_date = res.get('date') or res.get('year')
@@ -265,6 +280,7 @@ def generate_dataset(total_books_needed, progress=gr.Progress()):
                     if any(r['filename'].endswith(f"{id_}.txt") for r in records): 
                         continue
                     
+                    # The slow operation (download) is now outside the generator iteration
                     rec = download_book(id_, res.get('title', 'Unknown'), year, era_label, min_char_limit=min_chars)
                     if rec:
                         rec['topic'] = "Classic" if using_rescue else topic
@@ -272,9 +288,6 @@ def generate_dataset(total_books_needed, progress=gr.Progress()):
                         collected += 1
                         print(f"  ✅ Saved ({collected}/{books_per_era}): {rec['title']} ({year})")
                         
-                    if results_found >= (50 if era_label == "1_Late_Medieval" else (30 if is_hard_era else 10)):
-                        break
-                        
                 if results_found == 0:
                     print(f"  ⚠️ No results found for this query")
                     
@@ -284,6 +297,7 @@ def generate_dataset(total_books_needed, progress=gr.Progress()):
         
         print(f"Completed {era_label}: {collected}/{books_per_era} books collected")
         
+        # ... (rest of the fallback logic remains the same) ...
         if era_label == "1_Late_Medieval" and collected < books_per_era * 0.3:
             print(f"\n⚠️ EMERGENCY FALLBACK MODE for {era_label}")
             fallback_attempts = 0
@@ -300,10 +314,17 @@ def generate_dataset(total_books_needed, progress=gr.Progress()):
                 print(f"  > 🚨 Fallback #{fallback_attempts}: {term}")
                 
                 try:
-                    search = internetarchive.search_items(query, sorts=['downloads desc'], fields=['identifier', 'title', 'date', 'year'])
+                    search_generator = internetarchive.search_items(query, sorts=['downloads desc'], fields=['identifier', 'title', 'date', 'year'])
+                    
+                    # Pre-fetch for fallback as well
+                    fallback_batch = []
+                    for i, item in enumerate(search_generator):
+                        fallback_batch.append(item)
+                        if i >= 100: break # Increased limit for fallback
+                        
                     checked = 0
-                    for res in search:
-                        if collected >= books_per_era or checked >= 100:
+                    for res in fallback_batch:
+                        if collected >= books_per_era:
                             break
                         checked += 1
                         
@@ -324,6 +345,8 @@ def generate_dataset(total_books_needed, progress=gr.Progress()):
                 except Exception as e:
                     print(f"  ❌ Fallback error: {e}")
                     time.sleep(1)
+        # ... (end of fallback logic) ...
+
 
     if not records: return None, pd.DataFrame(), pd.DataFrame()
 
@@ -457,7 +480,6 @@ def train_model(dataset_path, epochs, batch_size, learning_rate, progress=gr.Pro
         )
         MODEL.to(DEVICE)
         
-        # FIX: Ensure we use the AdamW imported from torch.optim
         optimizer = AdamW(MODEL.parameters(), lr=learning_rate)
         total_steps = len(train_loader) * epochs
         scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_steps)