Update app.py

app.py

@@ -62,26 +62,26 @@ CAPTION_SERVERS = [
     "https://fred1012-fred1012-ptlc5u.hf.space/analyze",
     "https://fred1012-fred1012-u7lh57.hf.space/analyze",
     "https://fred1012-fred1012-q8djv1.hf.space/analyze",
-  "https://fredalone-fredalone-ozugrp.hf.space/analyze",
-  "https://fredalone-fredalone-9brxj2.hf.space/analyze",
-  "https://fredalone-fredalone-p8vq9a.hf.space/analyze",
-  "https://fredalone-fredalone-vbli2y.hf.space/analyze",
-  "https://fredalone-fredalone-uggger.hf.space/analyze",
-  "https://fredalone-fredalone-nmi7e8.hf.space/analyze",
-  "https://fredalone-fredalone-d1f26d.hf.space/analyze",
-  "https://fredalone-fredalone-461jp2.hf.space/analyze",
-  "https://fredalone-fredalone-3enfg4.hf.space/analyze",
-  "https://fredalone-fredalone-dqdbpv.hf.space/analyze",
-  "https://fredalone-fredalone-ivtjua.hf.space/analyze",
-  "https://fredalone-fredalone-6bezt2.hf.space/analyze",
-  "https://fredalone-fredalone-e0wfnk.hf.space/analyze",
-  "https://fredalone-fredalone-zu2t7j.hf.space/analyze",
-  "https://fredalone-fredalone-dqtv1o.hf.space/analyze",
-  "https://fredalone-fredalone-wclyog.hf.space/analyze",
-  "https://fredalone-fredalone-t27vig.hf.space/analyze",
-  "https://fredalone-fredalone-gahbxh.hf.space/analyze",
-  "https://fredalone-fredalone-kw2po4.hf.space/analyze",
-  "https://fredalone-fredalone-8h285h.hf.space/analyze"
+    "https://fredalone-fredalone-ozugrp.hf.space/analyze",
+    "https://fredalone-fredalone-9brxj2.hf.space/analyze",
+    "https://fredalone-fredalone-p8vq9a.hf.space/analyze",
+    "https://fredalone-fredalone-vbli2y.hf.space/analyze",
+    "https://fredalone-fredalone-uggger.hf.space/analyze",
+    "https://fredalone-fredalone-nmi7e8.hf.space/analyze",
+    "https://fredalone-fredalone-d1f26d.hf.space/analyze",
+    "https://fredalone-fredalone-461jp2.hf.space/analyze",
+    "https://fredalone-fredalone-3enfg4.hf.space/analyze",
+    "https://fredalone-fredalone-dqdbpv.hf.space/analyze",
+    "https://fredalone-fredalone-ivtjua.hf.space/analyze",
+    "https://fredalone-fredalone-6bezt2.hf.space/analyze",
+    "https://fredalone-fredalone-e0wfnk.hf.space/analyze",
+    "https://fredalone-fredalone-zu2t7j.hf.space/analyze",
+    "https://fredalone-fredalone-dqtv1o.hf.space/analyze",
+    "https://fredalone-fredalone-wclyog.hf.space/analyze",
+    "https://fredalone-fredalone-t27vig.hf.space/analyze",
+    "https://fredalone-fredalone-gahbxh.hf.space/analyze",
+    "https://fredalone-fredalone-kw2po4.hf.space/analyze",
+    "https://fredalone-fredalone-8h285h.hf.space/analyze"
 ]
 MODEL_TYPE = "Florence-2-large"  # Explicitly request large model
 DATA_COLLECTION_SERVER = "https://fred808-flow.hf.space"
@@ -350,7 +350,7 @@ async def get_model_info():
     return model_info
 
 async def process_image(server: CaptionServer, course: str, image: Dict) -> Dict:
-    """Process single image through one caption server"""
+    """Process single image through one caption server with better error handling"""
     if server.busy:
         return None
         
@@ -377,14 +377,21 @@ async def process_image(server: CaptionServer, course: str, image: Dict) -> Dict
             }
             print(f"Server {server.url} processed {image['filename']} in {processing_time:.2f}s ({server.fps:.2f} fps)")
             return metadata
+        else:
+            # Server responded but no caption (might be error or empty response)
+            error_msg = result.get('error', 'Unknown error') if result else 'No response'
+            print(f"Server {server.url} failed for {image['filename']}: {error_msg}")
+            return None
             
+    except asyncio.TimeoutError:
+        print(f"Server {server.url} timeout for {image['filename']}")
+        return None
     except Exception as e:
         print(f"Error processing {image['filename']} on {server.url}: {e}")
+        return None
         
     finally:
         server.busy = False
-        
-    return None
 
 async def submit_to_dataset(course: str, metadata_list: List[Dict]):
     """Submit course results to dataset collection server"""
@@ -417,7 +424,7 @@ async def submit_to_dataset(course: str, metadata_list: List[Dict]):
             return None
 
 async def process_course(course: str, servers: List[CaptionServer]):
-    """Process all images in a course using available servers"""
+    """Process all images in a course using available servers with retry logic"""
     if course not in processed_images:
         processed_images[course] = set()
     if course not in course_captions:
@@ -433,50 +440,87 @@ async def process_course(course: str, servers: List[CaptionServer]):
         return
     
     print(f"\nProcessing {len(images)} images for course {course}")
-    remaining_images = [img for img in images if img['filename'] not in processed_images[course]]
     
-    if not remaining_images:
+    # Track images that need processing with retry count (5 retries)
+    pending_images = {
+        img['filename']: {'image': img, 'retries': 0, 'max_retries': 5}
+        for img in images 
+        if img['filename'] not in processed_images[course]
+    }
+    
+    if not pending_images:
         print(f"All images already processed for course {course}")
         return
     
-    while remaining_images and is_processing:
+    failed_images = set()
+    total_images = len(images)
+    
+    while pending_images and is_processing:
         # Create tasks for each available server
         tasks = []
+        assigned_images = []
+        
         for server in servers:
-            if not server.busy and remaining_images:
-                img = remaining_images[0]
-                remaining_images = remaining_images[1:]
+            if not server.busy and pending_images:
+                # Get the next pending image
+                filename, img_data = next(iter(pending_images.items()))
+                img = img_data['image']
+                
+                # Assign this image to the server
                 tasks.append(process_image(server, course, img))
+                assigned_images.append((filename, img))
+                # Remove from pending temporarily while it's being processed
+                del pending_images[filename]
                 
         if not tasks:
+            # If no servers available, wait a bit
             await asyncio.sleep(0.1)
             continue
             
         # Process images in parallel across servers
         results = await asyncio.gather(*tasks)
         
-        # Handle results
+        # Handle results and retry logic
         has_new_results = False
-        for result in results:
+        for (filename, img), result in zip(assigned_images, results):
             if result:
-                processed_images[course].add(result['image'])
+                # Success - image was processed
+                processed_images[course].add(filename)
                 course_captions[course].append(result)
                 has_new_results = True
+                print(f"✓ Successfully processed {filename}")
+            else:
+                # Failure - check if we should retry
+                current_retries = assigned_images[0][0] if assigned_images else 0
+                img_data = {'image': img, 'retries': current_retries + 1, 'max_retries': 5}
+                
+                if img_data['retries'] <= img_data['max_retries']:
+                    # Put back in pending for retry
+                    pending_images[filename] = img_data
+                    print(f"↻ Retry {img_data['retries']}/{img_data['max_retries']} for {filename}")
+                else:
+                    # Max retries exceeded, mark as failed
+                    failed_images.add(filename)
+                    print(f"✗ Failed to process {filename} after {img_data['max_retries']} retries")
         
         # Save progress after each batch with new results
         if has_new_results:
             save_captions_to_file(course, course_captions[course])
         
         # Show progress
-        total = len(images)
         done = len(processed_images[course])
-        progress_percent = (done / total * 100) if total > 0 else 0
-        print(f"\rProgress: {done}/{total} images ({progress_percent:.1f}%)", end="")
+        pending_count = len(pending_images)
+        failed_count = len(failed_images)
+        progress_percent = (done / total_images * 100) if total_images > 0 else 0
+        print(f"\rProgress: {done}/{total_images} images ({progress_percent:.1f}%) - {pending_count} pending, {failed_count} failed", end="")
         
-        if not remaining_images and len(processed_images[course]) == len(images):
-            print(f"\nCourse {course} complete, submitting to dataset...")
-            await submit_to_dataset(course, course_captions[course])
-            # Don't clear the data, keep it for API queries
+        # Check if we're done (all images either processed or permanently failed)
+        if not pending_images:
+            if failed_images:
+                print(f"\nCourse {course} completed with {len(failed_images)} failed images: {list(failed_images)}")
+            else:
+                print(f"\nCourse {course} complete, submitting to dataset...")
+                await submit_to_dataset(course, course_captions[course])
             break
 
         # Small delay to prevent overwhelming the servers