Update main.py

main.py

@@ -64,6 +64,8 @@ CACHE_UPDATE_INTERVAL_SECONDS = int(os.getenv('CACHE_UPDATE_INTERVAL_SECONDS', 6
 # Webhook URLs
 SCAMMER_WEBHOOK_URL = os.getenv("SCAMMER_WEBHOOK_URL")
 VALUE_WEBHOOK_URL = os.getenv("VALUE_WEBHOOK_URL")
+# Optional: Separate webhook for dupe checks? Keep using the general one for now.
+DUPE_CHECK_WEBHOOK_URL = os.getenv("WEBHOOK_URL")
 
 
 # --- Global Cache ---
@@ -175,18 +177,19 @@ def format_currency(value: Any) -> Optional[str]:
         num = float(num_str)
         return f"${num:,.0f}"
     except (ValueError, TypeError):
-        if isinstance(value, str) and not re.match(r'^-?[\d,.$]+\$?$', value.strip()):
+        # Allow text like "Event", "Unobtainable" etc. to pass through
+        if isinstance(value, str) and value.strip() and not re.match(r'^-?[\d,.$]+\$?$', value.strip()):
              return value.strip() # Return original text if non-numeric-like
-        return 'N/A'
+        return 'N/A' # Return N/A for things that look like bad numbers
 
 def parse_cached_currency(value_str: Optional[str]) -> Optional[float]:
-    if value_str is None or value_str is None or str(value_str).strip().lower() == 'n/a':
+    if value_str is None or str(value_str).strip().lower() == 'n/a':
         return None
     try:
         num_str = str(value_str).replace('$', '').replace(',', '').strip()
         return float(num_str)
     except (ValueError, TypeError):
-        return None
+        return None # Return None if it's not a parsable number (e.g., "Event")
 
 def clean_string(value, default='N/A'):
     if value is None: return default
@@ -205,19 +208,20 @@ def parse_alt_accounts(value):
     return [acc.strip() for acc in raw_string.split(',') if acc.strip()]
 
 
-# --- Roblox API Helpers (Unchanged) ---
+# --- Roblox API Helpers ---
 async def get_roblox_user_id(session: aiohttp.ClientSession, username: str):
     if not username: return None
     url = "https://users.roblox.com/v1/usernames/users"
     payload = {"usernames": [username], "excludeBannedUsers": False}
     try:
-        async with session.post(url, json=payload) as response:
+        # Increased timeout specifically for Roblox API calls which can be slow
+        async with session.post(url, json=payload, timeout=aiohttp.ClientTimeout(total=10)) as response:
             if response.status == 200:
                 data = await response.json()
                 if data and data.get("data") and len(data["data"]) > 0:
                     return data["data"][0].get("id")
             else:
-                 logger.warning(f"Roblox API returned status {response.status} for username '{username}'")
+                 logger.warning(f"Roblox User API returned status {response.status} for username '{username}'")
             return None
     except asyncio.TimeoutError:
         logger.warning(f"Timeout fetching Roblox User ID for {username}")
@@ -233,7 +237,8 @@ async def get_roblox_avatar_url(session: aiohttp.ClientSession, user_id: int):
     if not user_id: return None
     url = f"https://thumbnails.roblox.com/v1/users/avatar-headshot?userIds={user_id}&size=150x150&format=Png&isCircular=false"
     try:
-        async with session.get(url) as response:
+         # Increased timeout specifically for Roblox API calls
+        async with session.get(url, timeout=aiohttp.ClientTimeout(total=10)) as response:
             if response.status == 200:
                 data = await response.json()
                 if data and data.get("data") and len(data["data"]) > 0:
@@ -253,29 +258,32 @@ async def get_roblox_avatar_url(session: aiohttp.ClientSession, user_id: int):
 
 
 # --- Data Processing Functions ---
-# These functions take raw rows from the sheet and process them.
 
 def process_sheet_data(values): # For Value Categories
     if not values: return []
     processed_data = []
-    for row in values: # Expected range like B6:P
+    for row_idx, row in enumerate(values): # Start counting from sheet row 6 (index 0 here)
         if not row or not any(str(cell).strip() for cell in row if cell is not None): continue
         # Skip header-like rows (e.g., "LEVEL 1 | HYPERCHROMES" in column F/index 4)
-        if len(row) > 4 and isinstance(row[4], str) and re.search(r'LEVEL \d+ \|', row[4]):
+        # Use index 4 for Value column (F)
+        if len(row) > 4 and isinstance(row[4], str) and re.search(r'(LEVEL \d+ \|)|(VALUE)', row[4], re.IGNORECASE):
+            #logger.debug(f"Skipping potential header row {row_idx+6}: {row}")
             continue
 
         # Indices based on B6:P (0-indexed from B)
         icon_formula = row[0] if len(row) > 0 else ''
-        name = row[2] if len(row) > 2 else 'N/A'
-        value_raw = row[4] if len(row) > 4 else 'N/A'
-        duped_value_raw = row[6] if len(row) > 6 else 'N/A'
-        market_value_raw = row[8] if len(row) > 8 else 'N/A'
-        demand = row[10] if len(row) > 10 else 'N/A'
-        notes = row[12] if len(row) > 12 else ''
+        name = row[2] if len(row) > 2 else 'N/A' # Column D
+        value_raw = row[4] if len(row) > 4 else 'N/A' # Column F
+        duped_value_raw = row[6] if len(row) > 6 else 'N/A' # Column H
+        market_value_raw = row[8] if len(row) > 8 else 'N/A' # Column J
+        demand = row[10] if len(row) > 10 else 'N/A' # Column L
+        notes = row[12] if len(row) > 12 else '' # Column N
         drive_url = row[14] if len(row) > 14 else None # Column P
 
         cleaned_name = clean_string(name)
-        if cleaned_name == 'N/A': # Skip rows without a valid name
+        # Also skip if name is clearly a header like "Name"
+        if cleaned_name == 'N/A' or cleaned_name.lower() == 'name':
+            #logger.debug(f"Skipping row {row_idx+6} due to missing/header name: {row}")
             continue
 
         processed_item = {
@@ -298,7 +306,11 @@ def process_user_scammer_data(values): # For User Scammer Sheet
         # Indices based on B6:G (0-indexed from B)
         discord_id = clean_string_optional(row[0]) if len(row) > 0 else None # Col B
         roblox_username = clean_string_optional(row[1]) if len(row) > 1 else None # Col C
+        # Skip if both identifiers are missing
         if not discord_id and not roblox_username: continue
+        # Skip if it looks like a header row
+        if str(discord_id).lower() == 'discord id' or str(roblox_username).lower() == 'roblox username':
+            continue
         processed_item = {
             'discord_id': discord_id,
             'roblox_username': roblox_username,
@@ -319,7 +331,11 @@ def process_server_scammer_data(values): # For Server Scammer Sheet
         # Indices based on B6:F (0-indexed from B)
         server_id = clean_string_optional(row[0]) if len(row) > 0 else None # Col B
         server_name = clean_string_optional(row[1]) if len(row) > 1 else None # Col C
+        # Skip if both identifiers are missing
         if not server_id and not server_name: continue
+        # Skip if it looks like a header row
+        if str(server_id).lower() == 'server id' or str(server_name).lower() == 'server name':
+             continue
         processed_item = {
             'server_id': server_id,
             'server_name': server_name,
@@ -334,12 +350,16 @@ def process_dwc_data(values): # For DWC Sheet
     if not values: return []
     processed_data = []
     for row in values: # Expected range like B6:G
-        if not row or len(row) < 3: continue
+        if not row or len(row) < 1: continue # Need at least one ID
         # Indices based on B6:G (0-indexed from B)
         user_id = clean_string_optional(row[0]) if len(row) > 0 else None # Col B
         server_id = clean_string_optional(row[1]) if len(row) > 1 else None # Col C
         roblox_user = clean_string_optional(row[2]) if len(row) > 2 else None # Col D
+        # Skip if all identifiers are missing
         if not user_id and not server_id and not roblox_user: continue
+        # Skip if it looks like a header row
+        if str(user_id).lower() == 'user id' or str(server_id).lower() == 'server id' or str(roblox_user).lower() == 'roblox user':
+             continue
         processed_item = {
             'status': 'DWC',
             'discord_user_id': user_id,
@@ -356,14 +376,21 @@ def process_dwc_data(values): # For DWC Sheet
 def process_dupe_list_data(values): # For Dupe List Sheet
     if not values: return []
     # Expected range like B2:B
-    return [row[0].strip().lower() for row in values if row and len(row)>0 and row[0] and isinstance(row[0], str) and row[0].strip()]
+    processed_dupes = []
+    for row in values:
+        if row and len(row)>0 and row[0] and isinstance(row[0], str):
+            username = row[0].strip().lower()
+            # Skip header or empty strings
+            if username and username not in ('username', 'usernames'):
+                 processed_dupes.append(username)
+    return processed_dupes
 
 
 # --- Async Fetching Functions ---
 
 async def fetch_batch_ranges_async(spreadsheet_id: str, ranges: List[str], value_render_option: str = 'FORMATTED_VALUE') -> List[Dict]:
     """Async wrapper to fetch multiple ranges using batchGet and return raw valueRanges."""
-    global sheets_service
+    global sheets_service, cache
     if not sheets_service:
         logger.warning(f"Attempted batch fetch from {spreadsheet_id} but Sheets service is unavailable.")
         raise Exception("Google Sheets service not initialized")
@@ -388,27 +415,41 @@ async def fetch_batch_ranges_async(spreadsheet_id: str, ranges: List[str], value
         return value_ranges # Return the raw list of valueRange objects
 
     except HttpError as e:
-        error_details = json.loads(e.content).get('error', {})
-        status = error_details.get('status')
-        message = error_details.get('message')
+        status_code = e.resp.status
+        error_details = {}
+        try:
+            error_details = json.loads(e.content).get('error', {})
+        except json.JSONDecodeError:
+             logger.error(f"Failed to parse JSON error content from Google API: {e.content}")
+
+        status = error_details.get('status', f'HTTP_{status_code}') # Use HTTP status if details missing
+        message = error_details.get('message', e._get_reason()) # Fallback message
         logger.error(f"Google API HTTP Error during batch fetch for {spreadsheet_id}: Status={status}, Message={message}")
-        # Handle potential API key/permission issues explicitly
-        if status == 'PERMISSION_DENIED' or status == 'UNAUTHENTICATED':
-            logger.critical(f"Authentication/Permission Error accessing {spreadsheet_id}. Please check credentials/API access.")
+
+        if status in ('PERMISSION_DENIED', 'UNAUTHENTICATED') or status_code == 403 or status_code == 401:
+            logger.critical(f"Authentication/Permission Error accessing {spreadsheet_id}. Disabling service checks.")
             cache["service_available"] = False # Mark service as down
-            sheets_service = None # Reset service
-        elif status == 'NOT_FOUND':
+            sheets_service = None # Reset service to force re-init attempt
+        elif status == 'NOT_FOUND' or status_code == 404:
              logger.error(f"Spreadsheet or Range not found error for {spreadsheet_id}. Ranges: {ranges}. Check IDs and Sheet Names.")
+        elif status_code >= 500: # Server-side errors on Google's end
+             logger.warning(f"Google API server error ({status_code}) for {spreadsheet_id}. May be temporary.")
+             # Keep service_available as True, retry might work
+        # else: # Other client errors (e.g., 400 Bad Request for invalid range format)
+
         raise e # Re-raise after logging
     except Exception as e:
         logger.error(f"Error during batch fetching from {spreadsheet_id} for ranges {ranges}: {e}", exc_info=True)
+        # Could be network issues, timeouts handled by aiohttp session typically
+        # Consider marking service unavailable for persistent non-HTTP errors too?
+        # cache["service_available"] = False # Optional: Be more aggressive
         raise e
 
 # --- Webhook Sending ---
 async def send_webhook_notification(session: aiohttp.ClientSession, webhook_url: str, embed: Dict):
     """Sends a Discord webhook notification with the provided embed."""
     if not webhook_url:
-        # logger.debug("Webhook URL not configured. Skipping notification.") # Optional: Log less verbosely
+        # logger.debug("Webhook URL not configured. Skipping notification.")
         return
     if not embed:
         logger.warning("Attempted to send webhook with empty embed.")
@@ -416,11 +457,14 @@ async def send_webhook_notification(session: aiohttp.ClientSession, webhook_url:
 
     webhook_data = {"embeds": [embed]}
     try:
+        # Use a reasonable timeout for webhook posts
         async with session.post(webhook_url, json=webhook_data, timeout=aiohttp.ClientTimeout(total=10)) as response:
             if response.status not in [200, 204]:
-                logger.warning(f"Failed to send webhook (Status: {response.status}): {await response.text()}")
-            # else: # Optional: Log success, can be verbose
-            #     logger.info(f"Webhook notification sent successfully to {webhook_url[:30]}...")
+                # Log more details on failure
+                response_text = await response.text()
+                logger.warning(f"Failed to send webhook to {webhook_url[:30]}... (Status: {response.status}): {response_text[:500]}") # Limit response text length
+            # else:
+            #     logger.debug(f"Webhook notification sent successfully to {webhook_url[:30]}...")
     except asyncio.TimeoutError:
         logger.warning(f"Timeout sending webhook to {webhook_url[:30]}...")
     except aiohttp.ClientError as e:
@@ -428,12 +472,13 @@ async def send_webhook_notification(session: aiohttp.ClientSession, webhook_url:
     except Exception as e:
         logger.error(f"Unexpected error sending webhook: {e}", exc_info=True)
 
-# --- Background Cache Update Task (Refactored for Batching & Webhooks) ---
+# --- Background Cache Update Task ---
 
 async def update_cache_periodically():
-    """Fetches data, processes, detects changes/new entries, sends webhooks, and updates cache."""
+    """Fetches data, processes, detects changes/new entries (if not first run), sends webhooks, and updates cache."""
     global cache
-    async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session: # Overall session timeout
+    # Increase overall session timeout slightly for robustness
+    async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=45)) as session:
         while True:
             if not cache["service_available"]:
                 logger.info("Attempting to re-initialize Google Sheets service...")
@@ -445,7 +490,7 @@ async def update_cache_periodically():
                 else:
                     logger.info("Google Sheets service re-initialized. Proceeding with cache update.")
 
-            logger.info("Starting cache update cycle...")
+            logger.info(f"Starting cache update cycle... (Cache Ready: {cache['is_ready']})")
             start_time = datetime.now(timezone.utc)
             webhook_tasks = [] # Store webhook sending tasks
 
@@ -507,6 +552,8 @@ async def update_cache_periodically():
                     if isinstance(result, Exception):
                         logger.error(f"Failed to fetch batch data for {key}: {result}")
                         current_errors[key] = str(result)
+                        # If fetch failed, likely service unavailable (handled by fetch_batch_ranges_async)
+                        # No need to explicitly set cache["service_available"] = False here again
                     else:
                         if key == "scammer_dwc_batch":
                             raw_scammer_dwc_results = result
@@ -537,6 +584,9 @@ async def update_cache_periodically():
                                 current_errors[f"process_{target_key}"] = str(e)
                         else:
                             logger.warning(f"No processor found for sheet name '{sheet_name}' derived from range '{range_str}' in Scammer/DWC sheet.")
+                else:
+                    logger.warning("Skipping Scammer/DWC processing due to fetch error.")
+
 
                 # --- Process Values/Dupes Results ---
                 if raw_values_dupes_results is not None:
@@ -565,39 +615,51 @@ async def update_cache_periodically():
                             target_key = "dupes" if sheet_name == DUPE_LIST_SHEET else f"values_{sheet_name}"
                             logger.error(f"Error processing data for {sheet_name}: {e}", exc_info=True)
                             current_errors[f"process_{target_key}"] = str(e)
+                else:
+                     logger.warning("Skipping Values/Dupes processing due to fetch error.")
 
                 # --- Fetch Roblox Avatars (for new data before comparison/webhook) ---
-                logger.info("Fetching Roblox avatars for newly processed data...")
-                avatar_tasks = []
-                # Combine lists needing avatars from the *newly fetched* data
-                entries_needing_avatars = new_cache_data.get("user_scammers", []) + new_cache_data.get("dwc", [])
-                for entry in entries_needing_avatars:
-                     if entry.get('roblox_username'):
-                         # Pass the specific entry dict to the update function
-                         avatar_tasks.append(fetch_avatar_for_entry_update(session, entry))
-                if avatar_tasks:
-                    await asyncio.gather(*avatar_tasks) # Exceptions logged within helper
-                logger.info(f"Finished fetching avatars for {len(avatar_tasks)} potential new entries.")
-
-                # --- Change Detection & Webhook Preparation (BEFORE Cache Update) ---
+                if not current_errors.get("scammer_dwc_batch") and \
+                   not current_errors.get("process_user_scammers") and \
+                   not current_errors.get("process_dwc"):
+                    logger.info("Fetching Roblox avatars for newly processed data...")
+                    avatar_tasks = []
+                    entries_needing_avatars = new_cache_data.get("user_scammers", []) + new_cache_data.get("dwc", [])
+                    for entry in entries_needing_avatars:
+                        if entry.get('roblox_username'):
+                            avatar_tasks.append(fetch_avatar_for_entry_update(session, entry))
+                    if avatar_tasks:
+                        await asyncio.gather(*avatar_tasks) # Exceptions logged within helper
+                    logger.info(f"Finished fetching avatars for {len(avatar_tasks)} potential new entries.")
+                else:
+                    logger.warning("Skipping avatar fetching due to errors in fetching/processing scammer/dwc data.")
+
+
+                # --- Change Detection & Webhook Preparation (ONLY if cache is ready) ---
                 current_time = datetime.now(timezone.utc)
                 timestamp_iso = current_time.isoformat()
+                detected_value_changes_for_api = {} # Always calculate for API, but only send webhooks if ready
+
+                # Perform comparisons regardless of cache readiness to populate detected_value_changes_for_api
+                # But only queue webhooks if cache["is_ready"] is True
 
-                # 1. Value Changes (Existing Logic + Webhook Prep)
-                detected_value_changes_for_api = {} # For the /api/value-changes endpoint
-                if "values" not in cache: cache["values"] = {} # Ensure exists for comparison
-                if VALUE_WEBHOOK_URL and not any(k.startswith("process_values_") for k in current_errors) and "values_dupes_batch" not in current_errors:
-                    logger.info("Detecting value changes for webhooks...")
+                # 1. Value Changes Calculation
+                if "values" not in cache: cache["values"] = {} # Ensure exists for comparison logic
+                if "values_dupes_batch" not in current_errors and not any(k.startswith("process_values_") for k in current_errors):
                     fields_to_compare = ['value', 'dupedValue', 'marketValue']
                     for category, new_items in fetched_values_categories.items():
-                        old_items_dict = {item['name']: item for item in cache["values"].get(category, [])}
+                        old_items_dict = {item['name'].lower(): item for item in cache["values"].get(category, [])} # Use lower case for comparison robustness
                         category_changes_for_api = []
 
                         for new_item in new_items:
                             item_name = new_item.get('name')
                             if not item_name or item_name == 'N/A': continue
+                            item_name_lower = item_name.lower()
+
+                            old_item = old_items_dict.get(item_name_lower)
+                            change_detected_for_webhook = False
+                            change_info_webhook = {}
 
-                            old_item = old_items_dict.get(item_name)
                             if old_item: # Check existing item for changes
                                 for field in fields_to_compare:
                                     old_val_str = old_item.get(field, 'N/A')
@@ -612,113 +674,133 @@ async def update_cache_periodically():
                                             "timestamp": timestamp_iso
                                         }
                                         category_changes_for_api.append(change_info)
+                                        change_detected_for_webhook = True
+                                        change_info_webhook = change_info # Store last change for potential webhook
+
+                            # Prepare webhook only if a change was found AND cache was ready
+                            if change_detected_for_webhook and cache["is_ready"] and VALUE_WEBHOOK_URL:
+                                embed = {
+                                    "title": f"Value Update: {item_name} ({category})",
+                                    "color": 3447003, # Blue
+                                    "fields": [
+                                        {"name": "Field Changed", "value": change_info_webhook['field'], "inline": True},
+                                        {"name": "Old Value", "value": f"`{change_info_webhook['old_value']}`", "inline": True},
+                                        {"name": "New Value", "value": f"`{change_info_webhook['new_value']}`", "inline": True},
+                                        {"name": "Item Notes", "value": new_item.get('notes', 'N/A')[:1020] or 'N/A', "inline": False}, # Limit notes length
+                                    ],
+                                    "timestamp": timestamp_iso
+                                }
+                                if new_item.get('icon'):
+                                    embed["thumbnail"] = {"url": new_item['icon']}
+                                webhook_tasks.append(send_webhook_notification(session, VALUE_WEBHOOK_URL, embed))
 
-                                        # Prepare webhook embed
-                                        embed = {
-                                            "title": f"Value Update: {item_name} ({category})",
-                                            "color": 3447003, # Blue
-                                            "fields": [
-                                                {"name": "Field Changed", "value": field, "inline": True},
-                                                {"name": "Old Value", "value": f"`{change_info['old_value']}`", "inline": True},
-                                                {"name": "New Value", "value": f"`{change_info['new_value']}`", "inline": True},
-                                                {"name": "Item Notes", "value": new_item.get('notes', 'N/A')[:1020] or 'N/A', "inline": False}, # Limit notes length
-                                            ],
-                                            "timestamp": timestamp_iso
-                                        }
-                                        if new_item.get('icon'):
-                                            embed["thumbnail"] = {"url": new_item['icon']}
-
-                                        webhook_tasks.append(send_webhook_notification(session, VALUE_WEBHOOK_URL, embed))
                         if category_changes_for_api:
                             detected_value_changes_for_api[category] = category_changes_for_api
-                    logger.info(f"Prepared {len(webhook_tasks)} value change webhooks.")
-                elif not VALUE_WEBHOOK_URL:
-                    logger.info("VALUE_WEBHOOK_URL not set, skipping value change webhook detection.")
+                    if cache["is_ready"] and VALUE_WEBHOOK_URL:
+                        logger.info(f"Prepared {len(webhook_tasks)} value change webhooks.")
+                    elif not VALUE_WEBHOOK_URL:
+                         logger.info("VALUE_WEBHOOK_URL not set, skipping value change webhook sending.")
+                    elif not cache["is_ready"]:
+                         logger.info("Skipping value change webhook sending during initial cache population.")
                 else:
-                    logger.warning("Skipping value change webhook detection due to fetch/processing errors.")
+                     logger.warning("Skipping value change detection and webhooks due to fetch/processing errors.")
 
 
-                # 2. New Scammers / DWC (New Logic + Webhook Prep)
-                if SCAMMER_WEBHOOK_URL and "scammer_dwc_batch" not in current_errors and not any(k.startswith("process_") and k in ["process_user_scammers", "process_server_scammers", "process_dwc"] for k in current_errors):
-                    logger.info("Detecting new scammer/DWC entries for webhooks...")
+                # 2. New Scammers / DWC (Only if cache is ready)
+                if cache["is_ready"] and SCAMMER_WEBHOOK_URL and \
+                   "scammer_dwc_batch" not in current_errors and \
+                   not any(k.startswith("process_") and k in ["process_user_scammers", "process_server_scammers", "process_dwc"] for k in current_errors):
+
+                    logger.info("Detecting new scammer/DWC entries for webhooks (cache is ready)...")
                     initial_webhook_task_count = len(webhook_tasks)
+                    added_new_scammer_webhook = False
+
+                    # --- User Scammers ---
+                    # Create keys robust to None values
+                    def get_user_scammer_key(item):
+                        return (item.get('discord_id') or 'none', item.get('roblox_username') or 'none')
+                    old_user_keys = set(get_user_scammer_key(item) for item in cache.get("user_scammers", []))
 
-                    # User Scammers
-                    old_user_keys = set((item.get('discord_id'), item.get('roblox_username')) for item in cache.get("user_scammers", []))
                     for item in new_cache_data.get("user_scammers", []):
-                        key = (item.get('discord_id'), item.get('roblox_username'))
+                        key = get_user_scammer_key(item)
                         if key not in old_user_keys:
                             logger.info(f"New User Scammer detected: Discord={item.get('discord_id')}, Roblox={item.get('roblox_username')}")
                             embed = {
-                                "title": "🚨 New User Scammer Added",
-                                "color": 15158332, # Red
+                                "title": "🚨 New User Scammer Added", "color": 15158332, # Red
                                 "fields": [
                                     {"name": "Discord ID", "value": f"`{item.get('discord_id', 'N/A')}`", "inline": True},
                                     {"name": "Roblox User", "value": f"`{item.get('roblox_username', 'N/A')}`", "inline": True},
-                                    {"name": "Scam Type", "value": item.get('scam_type', 'N/A'), "inline": False},
+                                    {"name": "Scam Type", "value": item.get('scam_type', 'N/A')[:1020] or 'N/A', "inline": False},
                                     {"name": "Explanation", "value": item.get('explanation', 'N/A')[:1020] or 'N/A', "inline": False},
-                                ],
-                                "timestamp": timestamp_iso
+                                ], "timestamp": timestamp_iso
                             }
-                            if item.get('evidence_link'):
-                                embed["fields"].append({"name": "Evidence", "value": item['evidence_link'], "inline": False})
-                            if item.get('alt_accounts'):
-                                embed["fields"].append({"name": "Alt Accounts", "value": ", ".join([f"`{a}`" for a in item['alt_accounts']]), "inline": False})
-                            if item.get('roblox_avatar_url'):
-                                embed["thumbnail"] = {"url": item['roblox_avatar_url']}
+                            if item.get('evidence_link'): embed["fields"].append({"name": "Evidence", "value": item['evidence_link'], "inline": False})
+                            if item.get('alt_accounts'): embed["fields"].append({"name": "Alt Accounts", "value": ", ".join([f"`{a}`" for a in item['alt_accounts']])[:1020] or 'N/A', "inline": False}) # Limit length
+                            if item.get('roblox_avatar_url'): embed["thumbnail"] = {"url": item['roblox_avatar_url']}
                             webhook_tasks.append(send_webhook_notification(session, SCAMMER_WEBHOOK_URL, embed))
+                            added_new_scammer_webhook = True
+
+                    # --- Server Scammers ---
+                    def get_server_scammer_key(item):
+                        return (item.get('server_id') or 'none', item.get('server_name') or 'none')
+                    old_server_keys = set(get_server_scammer_key(item) for item in cache.get("server_scammers", []))
 
-                    # Server Scammers
-                    old_server_keys = set((item.get('server_id'), item.get('server_name')) for item in cache.get("server_scammers", []))
                     for item in new_cache_data.get("server_scammers", []):
-                        key = (item.get('server_id'), item.get('server_name'))
+                        key = get_server_scammer_key(item)
                         if key not in old_server_keys:
                             logger.info(f"New Server Scammer detected: ID={item.get('server_id')}, Name={item.get('server_name')}")
                             embed = {
-                                "title": "🚨 New Server Scammer Added",
-                                "color": 15158332, # Red
+                                "title": "🚨 New Server Scammer Added", "color": 15158332, # Red
                                 "fields": [
                                     {"name": "Server ID", "value": f"`{item.get('server_id', 'N/A')}`", "inline": True},
                                     {"name": "Server Name", "value": f"`{item.get('server_name', 'N/A')}`", "inline": True},
-                                    {"name": "Scam Type", "value": item.get('scam_type', 'N/A'), "inline": False},
+                                    {"name": "Scam Type", "value": item.get('scam_type', 'N/A')[:1020] or 'N/A', "inline": False},
                                     {"name": "Explanation", "value": item.get('explanation', 'N/A')[:1020] or 'N/A', "inline": False},
-                                ],
-                                "timestamp": timestamp_iso
+                                ], "timestamp": timestamp_iso
                             }
-                            if item.get('evidence_link'):
-                                embed["fields"].append({"name": "Evidence", "value": item['evidence_link'], "inline": False})
+                            if item.get('evidence_link'): embed["fields"].append({"name": "Evidence", "value": item['evidence_link'], "inline": False})
                             webhook_tasks.append(send_webhook_notification(session, SCAMMER_WEBHOOK_URL, embed))
+                            added_new_scammer_webhook = True
+
+                    # --- DWC Entries ---
+                    def get_dwc_key(item):
+                         # Use a combination of available identifiers as the key
+                        return (
+                            item.get('discord_user_id') or 'none',
+                            item.get('discord_server_id') or 'none',
+                            item.get('roblox_username') or 'none'
+                        )
+                    old_dwc_keys = set(get_dwc_key(item) for item in cache.get("dwc", []))
 
-                    # DWC Entries
-                    old_dwc_keys = set((item.get('discord_user_id'), item.get('discord_server_id'), item.get('roblox_username')) for item in cache.get("dwc", []))
                     for item in new_cache_data.get("dwc", []):
-                        key = (item.get('discord_user_id'), item.get('discord_server_id'), item.get('roblox_username'))
+                        key = get_dwc_key(item)
                         if key not in old_dwc_keys:
                             logger.info(f"New DWC Entry detected: User={item.get('discord_user_id')}, Server={item.get('discord_server_id')}, Roblox={item.get('roblox_username')}")
                             embed = {
-                                "title": "⚠️ New DWC Entry Added",
-                                "color": 15105570, # Orange/Dark Yellow
+                                "title": "⚠️ New DWC Entry Added", "color": 15105570, # Orange/Dark Yellow
                                 "fields": [
                                     {"name": "Discord User ID", "value": f"`{item.get('discord_user_id', 'N/A')}`", "inline": True},
                                     {"name": "Discord Server ID", "value": f"`{item.get('discord_server_id', 'N/A')}`", "inline": True},
                                     {"name": "Roblox User", "value": f"`{item.get('roblox_username', 'N/A')}`", "inline": True},
                                     {"name": "Explanation", "value": item.get('explanation', 'N/A')[:1020] or 'N/A', "inline": False},
-                                ],
-                                "timestamp": timestamp_iso
+                                ], "timestamp": timestamp_iso
                             }
-                            if item.get('evidence_link'):
-                                embed["fields"].append({"name": "Evidence", "value": item['evidence_link'], "inline": False})
-                            if item.get('alt_accounts'):
-                                embed["fields"].append({"name": "Alt Accounts", "value": ", ".join([f"`{a}`" for a in item['alt_accounts']]), "inline": False})
-                            if item.get('roblox_avatar_url'):
-                                embed["thumbnail"] = {"url": item['roblox_avatar_url']}
+                            if item.get('evidence_link'): embed["fields"].append({"name": "Evidence", "value": item['evidence_link'], "inline": False})
+                            if item.get('alt_accounts'): embed["fields"].append({"name": "Alt Accounts", "value": ", ".join([f"`{a}`" for a in item['alt_accounts']])[:1020] or 'N/A', "inline": False})
+                            if item.get('roblox_avatar_url'): embed["thumbnail"] = {"url": item['roblox_avatar_url']}
                             webhook_tasks.append(send_webhook_notification(session, SCAMMER_WEBHOOK_URL, embed))
+                            added_new_scammer_webhook = True
 
-                    logger.info(f"Prepared {len(webhook_tasks) - initial_webhook_task_count} new scammer/DWC webhooks.")
+                    if added_new_scammer_webhook:
+                        logger.info(f"Prepared {len(webhook_tasks) - initial_webhook_task_count} new scammer/DWC webhooks.")
+
+                elif not cache["is_ready"]:
+                     logger.info("Skipping new scammer webhook detection during initial cache population.")
                 elif not SCAMMER_WEBHOOK_URL:
-                    logger.info("SCAMMER_WEBHOOK_URL not set, skipping new scammer webhook detection.")
-                else:
-                    logger.warning("Skipping new scammer webhook detection due to fetch/processing errors.")
+                     logger.info("SCAMMER_WEBHOOK_URL not set, skipping new scammer webhook detection.")
+                else: # Errors occurred
+                     logger.warning("Skipping new scammer webhook detection due to fetch/processing errors.")
+
 
                 # --- Send Webhooks Concurrently ---
                 if webhook_tasks:
@@ -726,11 +808,14 @@ async def update_cache_periodically():
                     await asyncio.gather(*webhook_tasks)
                     logger.info("Finished sending webhook notifications.")
                 else:
-                    logger.info("No webhooks to send for this cycle.")
+                    logger.info("No webhooks prepared to send for this cycle.")
 
 
                 # --- Final Cache Update ---
                 update_occurred = False
+                # Determine if this cycle *should* mark the cache as ready
+                can_set_ready = not cache["is_ready"] and not current_errors # Only set ready on first *fully successful* run
+
                 if not current_errors: # Perfect cycle
                     logger.info("Updating full cache (no errors during fetch or processing).")
                     cache["values"] = fetched_values_categories
@@ -740,7 +825,9 @@ async def update_cache_periodically():
                     cache["dupes"] = new_cache_data["dupes"]
                     cache["value_changes"] = detected_value_changes_for_api # Store the detected changes
                     cache["last_updated"] = current_time
-                    cache["is_ready"] = True
+                    if can_set_ready:
+                        logger.info("Marking cache as ready after initial successful population.")
+                        cache["is_ready"] = True
                     cache["service_available"] = True # Mark as available on success
                     update_occurred = True
                     logger.info(f"Cache update cycle completed successfully.")
@@ -750,7 +837,6 @@ async def update_cache_periodically():
 
                     # Update values only if the values/dupes batch succeeded AND processing succeeded
                     if "values_dupes_batch" not in current_errors and not any(k.startswith("process_values_") for k in current_errors):
-                        # Check if fetched data is different from cache before updating
                         if cache.get("values") != fetched_values_categories:
                              cache["values"] = fetched_values_categories
                              cache["value_changes"] = detected_value_changes_for_api # Update changes along with values
@@ -759,7 +845,6 @@ async def update_cache_periodically():
                     else:
                          logger.warning("Skipping update for 'values' due to errors.")
 
-
                     # Update dupes only if the values/dupes batch succeeded AND processing succeeded
                     if "values_dupes_batch" not in current_errors and "process_dupes" not in current_errors:
                          if cache.get("dupes") != new_cache_data["dupes"]:
@@ -783,15 +868,19 @@ async def update_cache_periodically():
                     else:
                          logger.warning("Skipping update for 'user_scammers', 'server_scammers', 'dwc' due to batch fetch error.")
 
-
                     if update_occurred:
                          cache["last_updated"] = current_time # Mark partial update time
-                         cache["is_ready"] = True # Allow access even if partial
-                         # Keep service_available as potentially false if there were fetch errors
-                         logger.info(f"Partially updated cache sections: {', '.join(partial_update_details)}")
+                         # Mark cache ready only if it was *already* ready and we managed a partial update
+                         # Or if this was the first run AND it was partially successful (maybe relax this?)
+                         # Let's stick to: only mark ready on first FULL success.
+                         if cache["is_ready"]: # If it was already ready, keep it ready
+                              logger.info(f"Partially updated cache sections: {', '.join(partial_update_details)}. Cache remains ready.")
+                         else:
+                              logger.info(f"Partially updated cache sections: {', '.join(partial_update_details)}. Cache remains NOT ready (requires full success on first run).")
+                         # Keep service_available based on whether fetch errors occurred
                     else:
                          logger.error(f"Cache update cycle failed, and no parts could be updated based on errors. Errors: {current_errors}")
-                         # Keep cache["is_ready"] as it was. Don't update timestamp.
+                         # Cache readiness and service availability remain unchanged
 
             except Exception as e:
                 logger.exception(f"Critical error during cache update cycle: {e}")
@@ -818,6 +907,8 @@ async def fetch_avatar_for_entry_update(session: aiohttp.ClientSession, entry: d
         user_id = await get_roblox_user_id(session, roblox_username)
         if user_id:
             new_avatar = await get_roblox_avatar_url(session, user_id)
+        # else: # User ID not found, keep avatar as None
+           # logger.debug(f"Roblox user ID not found for username: {roblox_username}")
 
     except Exception as e:
         # Log errors but don't stop the main update loop
@@ -825,7 +916,7 @@ async def fetch_avatar_for_entry_update(session: aiohttp.ClientSession, entry: d
         # Keep new_avatar as None on error
 
     finally:
-         # Update the entry dict directly (no need to check if changed, just set it)
+         # Update the entry dict directly
          entry['roblox_avatar_url'] = new_avatar
 
 
@@ -841,6 +932,8 @@ async def startup_event():
         logger.warning("SCAMMER_WEBHOOK_URL environment variable not set. New scammer notifications disabled.")
     if not VALUE_WEBHOOK_URL:
         logger.warning("VALUE_WEBHOOK_URL environment variable not set. Value change notifications disabled.")
+    if not DUPE_CHECK_WEBHOOK_URL:
+        logger.warning("WEBHOOK_URL (for dupe checks) environment variable not set. Dupe check notifications disabled.")
     asyncio.create_task(update_cache_periodically())
 
 
@@ -854,9 +947,7 @@ def check_cache_readiness():
               raise HTTPException(status_code=503, detail="Service temporarily unavailable due to backend connection issues. Please try again later.")
          else:
               raise HTTPException(status_code=503, detail="Cache is initializing or data is currently unavailable. Please try again shortly.")
-    # Optional: Add check for staleness?
-    # if cache["last_updated"] and (datetime.now(timezone.utc) - cache["last_updated"]).total_seconds() > CACHE_UPDATE_INTERVAL_SECONDS * 3:
-    #      raise HTTPException(status_code=503, detail="Data may be stale. Update in progress or backend issue.")
+
 
 @app.get("/")
 async def root():
@@ -894,8 +985,9 @@ async def get_category_values(category: str):
     matched_category = next((c for c in cache.get("values", {}).keys() if c.lower() == category.lower()), None)
     if not matched_category:
          # Check if the category *exists* conceptually even if empty
-         if category.lower() in [c.lower() for c in CATEGORIES]:
-              return {category: []} # Return empty list if category is valid but has no items
+         valid_categories_lower = [c.lower() for c in CATEGORIES]
+         if category.lower() in valid_categories_lower:
+              return {category: []} # Return empty list if category is valid but has no items yet
          else:
             raise HTTPException(status_code=404, detail=f"Category '{category}' not found.")
     return {matched_category: cache.get("values", {}).get(matched_category, [])}
@@ -909,7 +1001,8 @@ async def get_category_value_changes(category: str):
     matched_category = next((c for c in cache.get("value_changes", {}).keys() if c.lower() == category.lower()), None)
     if not matched_category:
          # Check if the category *exists* conceptually even if empty
-         if category.lower() in [c.lower() for c in CATEGORIES]:
+         valid_categories_lower = [c.lower() for c in CATEGORIES]
+         if category.lower() in valid_categories_lower:
               return {category: []} # Return empty list if category is valid but had no changes
          else:
             raise HTTPException(status_code=404, detail=f"Category '{category}' not found.")
@@ -943,7 +1036,7 @@ class UsernameCheck(BaseModel):
 
 @app.post("/api/check")
 async def check_username(data: UsernameCheck):
-    """Check if a username is duped using cached data and send webhook"""
+    """Check if a username is duped using cached data and optionally send webhook"""
     check_cache_readiness() # Use the standard readiness check
 
     username_to_check = data.username.strip().lower()
@@ -951,25 +1044,22 @@ async def check_username(data: UsernameCheck):
     dupes_list = cache.get("dupes", [])
     is_duped = username_to_check in dupes_list
 
-    # Webhook notification for checks resulting in "Not Found" (runs in background)
-    # Note: This uses the *value* webhook URL per original code, not the scammer one.
-    # If this needs to go to a *different* webhook, adjust the env var name.
+    # Webhook notification for checks resulting in "Not Found"
     if not is_duped:
-        webhook_url = os.getenv("WEBHOOK_URL") # Keep original env var name for this specific check
-        if webhook_url:
-            # Use a new session for this one-off task or pass the main one if safe
+        if DUPE_CHECK_WEBHOOK_URL:
             async def send_check_webhook():
                  try:
-                    async with aiohttp.ClientSession() as session: # Create a short-lived session
+                    # Use a short-lived session for this potentially frequent task
+                    async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=10)) as session:
                         embed = {
                                 "title": "User Dupe Check - Not Found",
                                 "description": f"Username `{data.username}` was checked against the dupe list but was **not** found.",
                                 "color": 16776960, # Yellow
                                 "timestamp": datetime.now(timezone.utc).isoformat()
                         }
-                        await send_webhook_notification(session, webhook_url, embed)
+                        await send_webhook_notification(session, DUPE_CHECK_WEBHOOK_URL, embed)
                  except Exception as e:
-                     logger.error(f"Error sending dupe check webhook: {e}") # Log errors from the task
+                     logger.error(f"Error sending dupe check webhook: {e}")
 
             asyncio.create_task(send_check_webhook()) # Fire and forget
         else: