update retry progress meter

app.py

@@ -66,74 +66,100 @@ def call_api(endpoint_url: str, payload: Dict[str, Any], max_retries: int = 5, r
     return {}
 
 def wake_servers(progress=gr.Progress()):
-    """Send wake-up requests to all endpoints"""
+    """Send wake-up requests to all endpoints in parallel with real-time updates"""
     results = {}
+    status_html = "<h3>Server Wake-up Results:</h3><ul>"
+    for name in ENDPOINTS.keys():
+        results[name] = "Pending..."
+        status_html += f"<li><strong>{name}</strong>: <span style='color:gray'>Pending...</span></li>"
+    status_html += "</ul>"
+    
+    # Initial status HTML
+    yield status_html
+    
+    def update_status_html():
+        """Generate HTML for current status"""
+        html = "<h3>Server Wake-up Results:</h3><ul>"
+        for name, status in results.items():
+            status_color = "green" if "Status: 200" in status else "red" if "Error" in status or "Failed" in status else "gray"
+            html += f"<li><strong>{name}</strong>: <span style='color:{status_color}'>{status}</span></li>"
+        html += "</ul>"
+        return html
     
-    def try_wake_endpoint(name, url, retry_count=0, max_retries=3):
+    def try_wake_endpoint(name, url):
         """Helper function to wake endpoint with retry logic"""
-        try:
-            # Send a small payload just to wake up the server
-            minimal_payload = {"inputs": "Hello"}
-            response = requests.post(
-                url, 
-                json=minimal_payload,
-                headers={"Authorization": f"Bearer {HF_TOKEN}"},
-                timeout=45
-            )
-            if response.status_code == 200:
-                return f"Status: {response.status_code}"
-            elif retry_count < max_retries:
-                # Calculate retry delay - 20 seconds for first retry, 5 seconds for subsequent retries
-                retry_delay = 20 if retry_count == 0 else 5
-                
-                # Update progress with retry information
-                progress_desc = f"Retrying {name} endpoint (attempt {retry_count+1}/{max_retries}, waiting {retry_delay}s)..."
-                progress(progress.value, desc=progress_desc)
-                print(progress_desc)
-                
-                # Wait before retrying
-                time.sleep(retry_delay)
-                
-                # Retry
-                return try_wake_endpoint(name, url, retry_count + 1, max_retries)
-            else:
-                # All retries failed
-                return f"Status: {response.status_code} (Failed after {max_retries} retries)"
-        except Exception as e:
-            if retry_count < max_retries:
-                # Calculate retry delay - 20 seconds for first retry, 5 seconds for subsequent retries
-                retry_delay = 20 if retry_count == 0 else 5
-                
-                # Update progress with retry information
-                progress_desc = f"Error connecting to {name} endpoint. Retrying (attempt {retry_count+1}/{max_retries}, waiting {retry_delay}s)..."
-                progress(progress.value, desc=progress_desc)
-                print(progress_desc)
+        retry_delays = [30, 20, 20, 20, 20, 20, 20, 20, 20, 20]  # Seconds to wait between retries
+        
+        for retry_count, retry_delay in enumerate(retry_delays):
+            try:
+                # Update status to show attempt
+                results[name] = f"Attempting to connect... (try {retry_count+1}/{len(retry_delays)+1})"
                 
-                # Wait before retrying
-                time.sleep(retry_delay)
+                # Send a small payload just to wake up the server
+                minimal_payload = {"inputs": "Hello"}
+                response = requests.post(
+                    url, 
+                    json=minimal_payload,
+                    headers={"Authorization": f"Bearer {HF_TOKEN}"},
+                    timeout=45
+                )
                 
-                # Retry
-                return try_wake_endpoint(name, url, retry_count + 1, max_retries)
-            else:
-                # All retries failed
-                return f"Error: {str(e)} (Failed after {max_retries} retries)"
+                if response.status_code == 200:
+                    results[name] = f"Status: {response.status_code}"
+                    return
+                else:
+                    # Non-200 response, prepare for retry
+                    if retry_count < len(retry_delays):
+                        results[name] = f"Status: {response.status_code}, retrying in {retry_delay}s... (attempt {retry_count+1}/{len(retry_delays)})"
+                        time.sleep(retry_delay)
+                    else:
+                        # All retries failed
+                        results[name] = f"Status: {response.status_code} (Failed after {len(retry_delays)} retries)"
+                        return
+            
+            except Exception as e:
+                # Connection error, prepare for retry
+                if retry_count < len(retry_delays):
+                    results[name] = f"Error connecting, retrying in {retry_delay}s... (attempt {retry_count+1}/{len(retry_delays)})"
+                    time.sleep(retry_delay)
+                else:
+                    # All retries failed
+                    results[name] = f"Error: {str(e)} (Failed after {len(retry_delays)} retries)"
+                    return
+    
+    # Function to process a single endpoint and update UI
+    def process_endpoint(name, url):
+        try:
+            try_wake_endpoint(name, url)
+        finally:
+            # Return the updated status HTML
+            return update_status_html()
     
-    total_endpoints = len(ENDPOINTS)
-    for i, (name, url) in enumerate(ENDPOINTS.items()):
-        progress(i/total_endpoints, desc=f"Waking up {name} endpoint...")
-        print(f"Waking up {name} endpoint...")
+    # Create a thread pool to wake up servers in parallel
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        # Start all tasks
+        futures = {executor.submit(process_endpoint, name, url): name 
+                  for name, url in ENDPOINTS.items()}
         
-        # Try to wake up the endpoint with retry logic
-        results[name] = try_wake_endpoint(name, url)
+        # Process results as they complete
+        for future in concurrent.futures.as_completed(futures):
+            name = futures[future]
+            try:
+                # Get the updated status HTML
+                status_html = future.result()
+                # Update progress
+                progress(sum(1 for r in results.values() if "Status: 200" in r) / len(ENDPOINTS), 
+                        desc=f"Waking up servers ({sum(1 for r in results.values() if r != 'Pending...')} of {len(ENDPOINTS)} processed)")
+                # Yield the updated status to show in real-time
+                yield status_html
+            except Exception as e:
+                print(f"Error processing {name}: {str(e)}")
+                results[name] = f"Error: Internal processing error"
+                yield update_status_html()
     
+    # Final update after all are complete
     progress(1.0, desc="Complete!")
-    status_html = "<h3>Server Wake-up Results:</h3><ul>"
-    for name, status in results.items():
-        status_color = "green" if "Status: 200" in status else "red"
-        status_html += f"<li><strong>{name}</strong>: <span style='color:{status_color}'>{status}</span></li>"
-    status_html += "</ul>"
-    
-    return status_html
+    yield update_status_html()
 
 def process_job_description(job_description: str) -> Dict:
     """Process job description and extract skills using the job endpoint"""
@@ -679,8 +705,9 @@ with gr.Blocks(title="Beyond Keywords: Resume Analysis System", js=js_func) as d
     gr.Markdown("# Beyond Keywords: Job Description and Resume Analyzer")
     gr.Markdown("Upload a job description and resume(s) to analyze skill matches and quality.")
     
-    wake_btn = gr.Button("Wake Servers (Do this first!)")
-    wake_status = gr.HTML(label="Server Status", value="<div style='color: #666;'>Click 'Wake Servers'<br>to initialize the system...</div>"); wake_btn.click(fn=wake_servers, inputs=None, outputs=wake_status)
+    wake_btn = gr.Button("Wake Servers (Do this first!) - Might take a 1-2 minutes.")
+    wake_status = gr.HTML(label="Server Status", value="<div style='color: #666;'>Click 'Wake Servers'<br>to initialize the system...</div>")
+    wake_btn.click(fn=wake_servers, inputs=None, outputs=wake_status)
     gr.Markdown("""<div style="height: 20px;"></div>""")
     
     with gr.Row():