Update app.py

app.py

@@ -14,7 +14,7 @@ ALL_REPO_ID = "samfred2/ALL"
 ATO_REPO_ID = "samfred2/ATO"
 OUTPUT_REPO_ID = "samfred2/ALL2"
 OUTPUT_DIR = "processed_files"
-HF_TOKEN = os.getenv("HF_TOKEN", "x")
+HF_TOKEN = os.getenv("HF_TOKEN", "")
 MAX_UPLOADS_PER_HOUR = 128
 RATE_LIMIT_DELAY = 3600  # 1 hour in seconds
 
@@ -92,10 +92,11 @@ def download_file(repo_id, filename, local_dir):
         logger.error(f"Could not download {filename}. Error: {e}")
         return None
 
-def upload_file_with_rate_limit(api, file_path, path_in_repo, upload_state):
+def upload_file_with_rate_limit(api, file_path, path_in_repo, upload_state, progress=None):
     """
     Uploads a file to HF with rate limiting (128 files per hour).
     Handles 429 errors by waiting an hour and resuming.
+    Tracks rate limit events in progress file.
     """
     while True:
         try:
@@ -106,6 +107,18 @@ def upload_file_with_rate_limit(api, file_path, path_in_repo, upload_state):
                     wait_time = RATE_LIMIT_DELAY - elapsed
                     wait_until = datetime.datetime.now() + datetime.timedelta(seconds=wait_time)
                     logger.info(f"Rate limit reached ({MAX_UPLOADS_PER_HOUR} uploads/hour). Waiting until {wait_until}")
+                    
+                    # Track rate limit event
+                    if progress is not None:
+                        event = {
+                            'timestamp': datetime.datetime.now().isoformat(),
+                            'type': 'hourly_limit_reached',
+                            'reason': f'Reached {MAX_UPLOADS_PER_HOUR} uploads/hour',
+                            'wait_seconds': int(wait_time),
+                            'resume_time': wait_until.isoformat()
+                        }
+                        progress.setdefault('rate_limit_events', []).append(event)
+                    
                     time.sleep(wait_time)
                     upload_state['hour_start'] = time.time()
                     upload_state['count'] = 0
@@ -125,6 +138,19 @@ def upload_file_with_rate_limit(api, file_path, path_in_repo, upload_state):
             if "429" in str(e) or "rate" in str(e).lower():
                 wait_until = datetime.datetime.now() + datetime.timedelta(seconds=RATE_LIMIT_DELAY)
                 logger.warning(f"Rate limit hit (429). Waiting 1 hour until {wait_until}")
+                
+                # Track rate limit error event
+                if progress is not None:
+                    event = {
+                        'timestamp': datetime.datetime.now().isoformat(),
+                        'type': 'http_429_error',
+                        'reason': 'HTTP 429 Too Many Requests from HF',
+                        'wait_seconds': RATE_LIMIT_DELAY,
+                        'resume_time': wait_until.isoformat(),
+                        'file': path_in_repo
+                    }
+                    progress.setdefault('rate_limit_events', []).append(event)
+                
                 time.sleep(RATE_LIMIT_DELAY)
                 upload_state['hour_start'] = time.time()
                 upload_state['count'] = 0
@@ -161,8 +187,8 @@ def load_progress(progress_file):
             with open(progress_file, 'r') as f:
                 return json.load(f)
         except:
-            return {'processed': [], 'uploaded': []}
-    return {'processed': [], 'uploaded': []}
+            return {'processed': [], 'uploaded': [], 'rate_limit_events': []}
+    return {'processed': [], 'uploaded': [], 'rate_limit_events': []}
 
 def save_progress(progress_file, progress):
     """Save progress tracking file."""
@@ -296,7 +322,7 @@ def process_datasets():
         
         # e. Upload to samfred2/ALL2
         if all_filename not in progress['uploaded']:
-            upload_file_with_rate_limit(api, final_output_path, all_filename, upload_state)
+            upload_file_with_rate_limit(api, final_output_path, all_filename, upload_state, progress)
             progress['uploaded'].append(all_filename)
             processing_state['uploaded'] += 1
         
@@ -347,7 +373,7 @@ def process_datasets():
         logger.info(f"Saved locally to {final_output_path}")
         
         # Upload to samfred2/ALL2
-        upload_file_with_rate_limit(api, final_output_path, all_filename, upload_state)
+        upload_file_with_rate_limit(api, final_output_path, all_filename, upload_state, progress)
         progress['uploaded'].append(all_filename)
         processing_state['uploaded'] += 1