timeout and retry for slack

main.py

@@ -3,6 +3,7 @@ import os
 import requests
 import json
 import logging
+import time
 
 # Set up logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -17,19 +18,76 @@ MODEL_CATALOG_WEBHOOK_SECRET = os.environ.get("MODEL_CATALOG_WEBHOOK_SECRET")
 SIMSHIP_WEBHOOK = os.environ.get("SIMSHIP_WEBHOOK")
 SIMSHIP_WEBHOOK_SECRET = os.environ.get("SIMSHIP_WEBHOOK_SECRET")
 
-def send_slack_message(message: str):
-    """Send a message to Slack using webhook URL"""
+def send_slack_message(message: str, max_retries: int = 3, timeout: int = 10, retry_delay: int = 2):
+    """
+    Send a message to Slack using webhook URL with timeout and retry mechanism
+    
+    Args:
+        message: The message to send to Slack
+        max_retries: Maximum number of retry attempts (default: 3)
+        timeout: Request timeout in seconds (default: 30)
+        retry_delay: Delay between retries in seconds (default: 2)
+    """
     if not SLACK_WEBHOOK_URL:
         logger.warning(f"No Slack webhook URL configured. Message: {message}")
         return
     
     payload = {"text": message}
-    try:
-        response = requests.post(SLACK_WEBHOOK_URL, json=payload)
-        response.raise_for_status()
-        logger.info(f"Slack message sent successfully: {message}")
-    except requests.exceptions.RequestException as e:
-        logger.error(f"Failed to send Slack message: {e}")
+    
+    for attempt in range(max_retries + 1):  # +1 for initial attempt
+        try:
+            logger.info(f"Sending Slack message (attempt {attempt + 1}/{max_retries + 1})")
+            response = requests.post(
+                SLACK_WEBHOOK_URL, 
+                json=payload, 
+                timeout=timeout
+            )
+            response.raise_for_status()
+            logger.info(f"Slack message sent successfully: {message}")
+            return  # Success, exit the function
+            
+        except requests.exceptions.Timeout as e:
+            logger.warning(f"Slack request timeout on attempt {attempt + 1}: {e}")
+            if attempt < max_retries:
+                logger.info(f"Retrying in {retry_delay} seconds...")
+                time.sleep(retry_delay)
+            else:
+                logger.error(f"Failed to send Slack message after {max_retries + 1} attempts due to timeout")
+                
+        except requests.exceptions.HTTPError as e:
+            # For HTTP errors, check if it's worth retrying
+            if hasattr(e.response, 'status_code'):
+                status_code = e.response.status_code
+                if status_code in [429, 500, 502, 503, 504]:  # Retry-able errors
+                    logger.warning(f"Slack HTTP error {status_code} on attempt {attempt + 1}: {e}")
+                    if attempt < max_retries:
+                        logger.info(f"Retrying in {retry_delay} seconds...")
+                        time.sleep(retry_delay)
+                    else:
+                        logger.error(f"Failed to send Slack message after {max_retries + 1} attempts due to HTTP error {status_code}")
+                else:
+                    # Non-retryable HTTP error (4xx except 429)
+                    logger.error(f"Failed to send Slack message due to non-retryable HTTP error {status_code}: {e}")
+                    break
+            else:
+                logger.error(f"Failed to send Slack message due to HTTP error: {e}")
+                break
+                
+        except requests.exceptions.ConnectionError as e:
+            logger.warning(f"Slack connection error on attempt {attempt + 1}: {e}")
+            if attempt < max_retries:
+                logger.info(f"Retrying in {retry_delay} seconds...")
+                time.sleep(retry_delay)
+            else:
+                logger.error(f"Failed to send Slack message after {max_retries + 1} attempts due to connection error")
+                
+        except requests.exceptions.RequestException as e:
+            logger.warning(f"Slack request error on attempt {attempt + 1}: {e}")
+            if attempt < max_retries:
+                logger.info(f"Retrying in {retry_delay} seconds...")
+                time.sleep(retry_delay)
+            else:
+                logger.error(f"Failed to send Slack message after {max_retries + 1} attempts due to request error")
 
 def process_model_catalog_webhook(data: dict):
     """