Update main.py

main.py

@@ -1,6 +1,3 @@
-# --- START OF FILE main.py ---
-
-# main.py
 from fastapi import FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
@@ -37,9 +34,9 @@ app.add_middleware(
 
 # Google Sheets Config
 # Spreadsheet containing Scammer and DWC info
-SCAMMER_DWC_SPREADSHEET_ID = '1sgkhBNGw_r6tBIxvdeXaI0bVmWBeACN4jiw_oDEeXLw'
+SCAMMER_DWC_SPREADSHEET_ID = os.getenv('SCAMMER_DWC_SPREADSHEET_ID', '1sgkhBNGw_r6tBIxvdeXaI0bVmWBeACN4jiw_oDEeXLw')
 # Spreadsheet containing Value lists and Dupe list
-VALUES_DUPE_SPREADSHEET_ID = '1Toe07o3P517q8sm9Qb1e5xyFWCuwgskj71IKJwJNfNU'
+VALUES_DUPE_SPREADSHEET_ID = os.getenv('VALUES_DUPE_SPREADSHEET_ID', '1Toe07o3P517q8sm9Qb1e5xyFWCuwgskj71IKJwJNfNU')
 
 SCOPES = ['https://www.googleapis.com/auth/spreadsheets.readonly']
 
@@ -62,12 +59,17 @@ CATEGORIES = [
 VALUES_RANGE = 'B6:P' # Range within each category sheet
 
 # Cache Update Interval
-CACHE_UPDATE_INTERVAL_SECONDS = 60 * 5 # 5 minutes
+CACHE_UPDATE_INTERVAL_SECONDS = int(os.getenv('CACHE_UPDATE_INTERVAL_SECONDS', 60 * 5)) # 5 minutes default
+
+# Webhook URLs
+SCAMMER_WEBHOOK_URL = os.getenv("SCAMMER_WEBHOOK_URL")
+VALUE_WEBHOOK_URL = os.getenv("VALUE_WEBHOOK_URL")
+
 
 # --- Global Cache ---
 cache = {
     "values": {},             # Dict mapping category name to list of items
-    "value_changes": {},      # Dict mapping category name to list of changes
+    "value_changes": {},      # Dict mapping category name to list of recent changes (for API endpoint)
     "user_scammers": [],
     "server_scammers": [],
     "dwc": [],
@@ -91,35 +93,44 @@ def quote_sheet_name(name: str) -> str:
     return name
 
 def init_google_sheets(scopes=SCOPES):
-    """Initialize Google Sheets credentials from environment variable"""
+    """Initialize Google Sheets credentials from environment variable or file"""
     global sheets_service, cache
     try:
         creds_json_str = os.getenv('CREDENTIALS_JSON')
-        if not creds_json_str:
-            logger.error("CREDENTIALS_JSON environment variable not found")
-            raise ValueError("CREDENTIALS_JSON environment variable not found")
-        creds_json = json.loads(creds_json_str)
-        creds = ServiceAccountCredentials.from_service_account_info(
-            creds_json,
-            scopes=scopes
-        )
-        sheets_service = build('sheets', 'v4', credentials=creds, cache_discovery=False) # Disable discovery cache
-        logger.info("Google Sheets service initialized successfully from ENV VAR.")
-        cache["service_available"] = True
-        return sheets_service
-    except Exception as e:
-        logger.error(f"Error initializing Google Sheets from ENV VAR: {e}")
-        # Fallback attempt
-        try:
-            logger.info("Falling back to loading credentials from file 'credentials.json'")
-            creds = ServiceAccountCredentials.from_service_account_file(
-                'credentials.json',
+        if creds_json_str:
+            logger.info("Attempting to load credentials from CREDENTIALS_JSON environment variable.")
+            creds_json = json.loads(creds_json_str)
+            creds = ServiceAccountCredentials.from_service_account_info(
+                creds_json,
                 scopes=scopes
             )
-            sheets_service = build('sheets', 'v4', credentials=creds, cache_discovery=False)
-            logger.info("Google Sheets service initialized successfully from file.")
+            sheets_service = build('sheets', 'v4', credentials=creds, cache_discovery=False) # Disable discovery cache
+            logger.info("Google Sheets service initialized successfully from ENV VAR.")
             cache["service_available"] = True
             return sheets_service
+        else:
+            logger.info("CREDENTIALS_JSON environment variable not found. Falling back to file.")
+            raise ValueError("CREDENTIALS_JSON not set") # Trigger fallback explicitly
+
+    except Exception as e:
+        logger.warning(f"Error initializing Google Sheets from ENV VAR: {e}. Trying file...")
+        try:
+            # Fallback to loading credentials from file 'credentials.json'
+            creds_file = 'credentials.json'
+            if os.path.exists(creds_file):
+                 logger.info(f"Attempting to load credentials from file '{creds_file}'")
+                 creds = ServiceAccountCredentials.from_service_account_file(
+                    creds_file,
+                    scopes=scopes
+                 )
+                 sheets_service = build('sheets', 'v4', credentials=creds, cache_discovery=False)
+                 logger.info("Google Sheets service initialized successfully from file.")
+                 cache["service_available"] = True
+                 return sheets_service
+            else:
+                 logger.error(f"Credentials file '{creds_file}' not found.")
+                 raise FileNotFoundError(f"'{creds_file}' not found")
+
         except Exception as file_e:
             logger.error(f"Error loading credentials from file: {file_e}")
             logger.critical("Google Sheets service could not be initialized. API will be limited.")
@@ -154,8 +165,6 @@ def extract_image_url(formula, drive_url=None):
     if formula.startswith('=IMAGE('):
         match = re.search(r'=IMAGE\("([^"]+)"', formula)
         if match: return match.group(1)
-    # If it wasn't a formula or direct URL, and no drive_url, return empty or original?
-    # Let's assume if it's not a recognizable URL/formula, it's not an image source.
     return '' # Return empty string if no valid URL found
 
 def format_currency(value: Any) -> Optional[str]:
@@ -207,6 +216,8 @@ async def get_roblox_user_id(session: aiohttp.ClientSession, username: str):
                 data = await response.json()
                 if data and data.get("data") and len(data["data"]) > 0:
                     return data["data"][0].get("id")
+            else:
+                 logger.warning(f"Roblox API returned status {response.status} for username '{username}'")
             return None
     except asyncio.TimeoutError:
         logger.warning(f"Timeout fetching Roblox User ID for {username}")
@@ -215,7 +226,7 @@ async def get_roblox_user_id(session: aiohttp.ClientSession, username: str):
         logger.warning(f"Network error fetching Roblox User ID for {username}: {e}")
         return None
     except Exception as e:
-        logger.error(f"Unexpected exception fetching Roblox User ID for {username}: {e}")
+        logger.error(f"Unexpected exception fetching Roblox User ID for {username}: {e}", exc_info=True)
         return None
 
 async def get_roblox_avatar_url(session: aiohttp.ClientSession, user_id: int):
@@ -227,6 +238,8 @@ async def get_roblox_avatar_url(session: aiohttp.ClientSession, user_id: int):
                 data = await response.json()
                 if data and data.get("data") and len(data["data"]) > 0:
                     return data["data"][0].get("imageUrl")
+            else:
+                 logger.warning(f"Roblox Thumbnail API returned status {response.status} for user ID {user_id}")
             return None
     except asyncio.TimeoutError:
         logger.warning(f"Timeout fetching Roblox avatar for User ID {user_id}")
@@ -235,22 +248,23 @@ async def get_roblox_avatar_url(session: aiohttp.ClientSession, user_id: int):
         logger.warning(f"Network error fetching Roblox avatar for User ID {user_id}: {e}")
         return None
     except Exception as e:
-        logger.error(f"Unexpected exception fetching Roblox avatar for User ID {user_id}: {e}")
+        logger.error(f"Unexpected exception fetching Roblox avatar for User ID {user_id}: {e}", exc_info=True)
         return None
 
 
 # --- Data Processing Functions ---
 # These functions take raw rows from the sheet and process them.
-# They are now independent of *which* sheet they came from, as long as the structure matches.
 
 def process_sheet_data(values): # For Value Categories
     if not values: return []
     processed_data = []
     for row in values: # Expected range like B6:P
         if not row or not any(str(cell).strip() for cell in row if cell is not None): continue
+        # Skip header-like rows (e.g., "LEVEL 1 | HYPERCHROMES" in column F/index 4)
+        if len(row) > 4 and isinstance(row[4], str) and re.search(r'LEVEL \d+ \|', row[4]):
+            continue
 
         # Indices based on B6:P (0-indexed from B)
-        # B=0, C=1, D=2, E=3, F=4, G=5, H=6, I=7, J=8, K=9, L=10, M=11, N=12, O=13, P=14
         icon_formula = row[0] if len(row) > 0 else ''
         name = row[2] if len(row) > 2 else 'N/A'
         value_raw = row[4] if len(row) > 4 else 'N/A'
@@ -260,15 +274,13 @@ def process_sheet_data(values): # For Value Categories
         notes = row[12] if len(row) > 12 else ''
         drive_url = row[14] if len(row) > 14 else None # Column P
 
-        # Skip header-like rows (e.g., "LEVEL 1 | HYPERCHROMES" in column F/index 4)
-        if len(row) > 4 and isinstance(row[4], str) and re.search(r'LEVEL \d+ \|', row[4]):
-            continue
-        if clean_string(name) == 'N/A':
+        cleaned_name = clean_string(name)
+        if cleaned_name == 'N/A': # Skip rows without a valid name
             continue
 
         processed_item = {
             'icon': extract_image_url(icon_formula, drive_url),
-            'name': clean_string(name, 'N/A'),
+            'name': cleaned_name,
             'value': format_currency(value_raw),
             'dupedValue': format_currency(duped_value_raw),
             'marketValue': format_currency(market_value_raw),
@@ -284,7 +296,6 @@ def process_user_scammer_data(values): # For User Scammer Sheet
     for row in values: # Expected range like B6:G
         if not row or len(row) < 2: continue
         # Indices based on B6:G (0-indexed from B)
-        # B=0, C=1, D=2, E=3, F=4, G=5
         discord_id = clean_string_optional(row[0]) if len(row) > 0 else None # Col B
         roblox_username = clean_string_optional(row[1]) if len(row) > 1 else None # Col C
         if not discord_id and not roblox_username: continue
@@ -295,7 +306,7 @@ def process_user_scammer_data(values): # For User Scammer Sheet
             'explanation': clean_string(row[3]) if len(row) > 3 else 'N/A', # Col E
             'evidence_link': clean_string_optional(row[4]) if len(row) > 4 else None, # Col F
             'alt_accounts': parse_alt_accounts(row[5]) if len(row) > 5 else [], # Col G
-            'roblox_avatar_url': None
+            'roblox_avatar_url': None # Will be filled later
         }
         processed_data.append(processed_item)
     return processed_data
@@ -306,7 +317,6 @@ def process_server_scammer_data(values): # For Server Scammer Sheet
     for row in values: # Expected range like B6:F
         if not row or len(row) < 2: continue
         # Indices based on B6:F (0-indexed from B)
-        # B=0, C=1, D=2, E=3, F=4
         server_id = clean_string_optional(row[0]) if len(row) > 0 else None # Col B
         server_name = clean_string_optional(row[1]) if len(row) > 1 else None # Col C
         if not server_id and not server_name: continue
@@ -326,7 +336,6 @@ def process_dwc_data(values): # For DWC Sheet
     for row in values: # Expected range like B6:G
         if not row or len(row) < 3: continue
         # Indices based on B6:G (0-indexed from B)
-        # B=0, C=1, D=2, E=3, F=4, G=5
         user_id = clean_string_optional(row[0]) if len(row) > 0 else None # Col B
         server_id = clean_string_optional(row[1]) if len(row) > 1 else None # Col C
         roblox_user = clean_string_optional(row[2]) if len(row) > 2 else None # Col D
@@ -339,7 +348,7 @@ def process_dwc_data(values): # For DWC Sheet
             'explanation': clean_string(row[3]) if len(row) > 3 else 'N/A', # Col E
             'evidence_link': clean_string_optional(row[4]) if len(row) > 4 else None, # Col F
             'alt_accounts': parse_alt_accounts(row[5]) if len(row) > 5 else [], # Col G
-            'roblox_avatar_url': None
+            'roblox_avatar_url': None # Will be filled later
         }
         processed_data.append(processed_item)
     return processed_data
@@ -383,30 +392,62 @@ async def fetch_batch_ranges_async(spreadsheet_id: str, ranges: List[str], value
         status = error_details.get('status')
         message = error_details.get('message')
         logger.error(f"Google API HTTP Error during batch fetch for {spreadsheet_id}: Status={status}, Message={message}")
-        raise e
+        # Handle potential API key/permission issues explicitly
+        if status == 'PERMISSION_DENIED' or status == 'UNAUTHENTICATED':
+            logger.critical(f"Authentication/Permission Error accessing {spreadsheet_id}. Please check credentials/API access.")
+            cache["service_available"] = False # Mark service as down
+            sheets_service = None # Reset service
+        elif status == 'NOT_FOUND':
+             logger.error(f"Spreadsheet or Range not found error for {spreadsheet_id}. Ranges: {ranges}. Check IDs and Sheet Names.")
+        raise e # Re-raise after logging
     except Exception as e:
-        logger.error(f"Error during batch fetching from {spreadsheet_id} for ranges {ranges}: {e}")
+        logger.error(f"Error during batch fetching from {spreadsheet_id} for ranges {ranges}: {e}", exc_info=True)
         raise e
 
-# --- Background Cache Update Task (Refactored for Batching per Spreadsheet) ---
+# --- Webhook Sending ---
+async def send_webhook_notification(session: aiohttp.ClientSession, webhook_url: str, embed: Dict):
+    """Sends a Discord webhook notification with the provided embed."""
+    if not webhook_url:
+        # logger.debug("Webhook URL not configured. Skipping notification.") # Optional: Log less verbosely
+        return
+    if not embed:
+        logger.warning("Attempted to send webhook with empty embed.")
+        return
+
+    webhook_data = {"embeds": [embed]}
+    try:
+        async with session.post(webhook_url, json=webhook_data, timeout=aiohttp.ClientTimeout(total=10)) as response:
+            if response.status not in [200, 204]:
+                logger.warning(f"Failed to send webhook (Status: {response.status}): {await response.text()}")
+            # else: # Optional: Log success, can be verbose
+            #     logger.info(f"Webhook notification sent successfully to {webhook_url[:30]}...")
+    except asyncio.TimeoutError:
+        logger.warning(f"Timeout sending webhook to {webhook_url[:30]}...")
+    except aiohttp.ClientError as e:
+        logger.error(f"Network error sending webhook to {webhook_url[:30]}...: {e}")
+    except Exception as e:
+        logger.error(f"Unexpected error sending webhook: {e}", exc_info=True)
+
+# --- Background Cache Update Task (Refactored for Batching & Webhooks) ---
 
 async def update_cache_periodically():
-    """Fetches data using batchGet per spreadsheet, processes, detects changes, and updates cache."""
+    """Fetches data, processes, detects changes/new entries, sends webhooks, and updates cache."""
     global cache
-    async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=25)) as session: # Slightly longer timeout
+    async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session: # Overall session timeout
         while True:
             if not cache["service_available"]:
                 logger.info("Attempting to re-initialize Google Sheets service...")
                 init_google_sheets()
                 if not cache["service_available"]:
                     logger.warning("Google Sheets service still unavailable, skipping cache update cycle.")
-                    await asyncio.sleep(CACHE_UPDATE_INTERVAL_SECONDS * 2)
+                    await asyncio.sleep(CACHE_UPDATE_INTERVAL_SECONDS * 2) # Wait longer if service is down
                     continue
                 else:
                     logger.info("Google Sheets service re-initialized. Proceeding with cache update.")
 
             logger.info("Starting cache update cycle...")
             start_time = datetime.now(timezone.utc)
+            webhook_tasks = [] # Store webhook sending tasks
 
             # Prepare temporary storage for fetched data
             fetched_values_categories = {} # { "CategoryName": [items...] }
@@ -420,7 +461,6 @@ async def update_cache_periodically():
 
             try:
                 # --- Define Ranges and Processors ---
-                # Scammer/DWC Spreadsheet
                 scammer_dwc_ranges = [
                     f"{quote_sheet_name(USER_SCAMMER_SHEET)}!{USER_SCAMMER_RANGE}",
                     f"{quote_sheet_name(SERVER_SCAMMER_SHEET)}!{SERVER_SCAMMER_RANGE}",
@@ -431,13 +471,12 @@ async def update_cache_periodically():
                     SERVER_SCAMMER_SHEET: process_server_scammer_data,
                     DWC_SHEET: process_dwc_data,
                 }
-                scammer_dwc_target_key_map = { # Map sheet name to cache key
+                scammer_dwc_target_key_map = {
                     USER_SCAMMER_SHEET: "user_scammers",
                     SERVER_SCAMMER_SHEET: "server_scammers",
                     DWC_SHEET: "dwc",
                 }
 
-                # Values/Dupes Spreadsheet
                 values_dupes_ranges = [f"{quote_sheet_name(DUPE_LIST_SHEET)}!{DUPE_LIST_RANGE}"]
                 values_dupes_ranges.extend([f"{quote_sheet_name(cat)}!{VALUES_RANGE}" for cat in CATEGORIES])
 
@@ -446,12 +485,12 @@ async def update_cache_periodically():
                     "scammer_dwc_batch": fetch_batch_ranges_async(
                         SCAMMER_DWC_SPREADSHEET_ID,
                         scammer_dwc_ranges,
-                        value_render_option='FORMATTED_VALUE' # These don't need formulas
+                        value_render_option='FORMATTED_VALUE'
                     ),
                     "values_dupes_batch": fetch_batch_ranges_async(
                         VALUES_DUPE_SPREADSHEET_ID,
                         values_dupes_ranges,
-                        value_render_option='FORMATTED_VALUE' # These don't need formulas
+                        value_render_option='FORMATTED_VALUE'
                     )
                 }
 
@@ -469,7 +508,6 @@ async def update_cache_periodically():
                         logger.error(f"Failed to fetch batch data for {key}: {result}")
                         current_errors[key] = str(result)
                     else:
-                        # Store the raw valueRanges list
                         if key == "scammer_dwc_batch":
                             raw_scammer_dwc_results = result
                         elif key == "values_dupes_batch":
@@ -480,12 +518,11 @@ async def update_cache_periodically():
                     logger.info(f"Processing {len(raw_scammer_dwc_results)} valueRanges from Scammer/DWC sheet...")
                     for vr in raw_scammer_dwc_results:
                         range_str = vr.get('range', '')
-                        # Extract sheet name (handle quotes)
                         match = re.match(r"^'?([^'!]+)'?!", range_str)
                         if not match:
                             logger.warning(f"Could not extract sheet name from range '{range_str}' in Scammer/DWC response.")
                             continue
-                        sheet_name = match.group(1).replace("''", "'") # Unescape quotes
+                        sheet_name = match.group(1).replace("''", "'")
 
                         if sheet_name in scammer_dwc_processor_map:
                             processor = scammer_dwc_processor_map[sheet_name]
@@ -493,7 +530,7 @@ async def update_cache_periodically():
                             values = vr.get('values', [])
                             try:
                                 processed_data = processor(values)
-                                new_cache_data[target_key] = processed_data
+                                new_cache_data[target_key] = processed_data # Store fetched data temporarily
                                 logger.info(f"Processed {len(processed_data)} items for {sheet_name} -> {target_key}")
                             except Exception as e:
                                 logger.error(f"Error processing data for {sheet_name} using {processor.__name__}: {e}", exc_info=True)
@@ -516,11 +553,11 @@ async def update_cache_periodically():
                         try:
                             if sheet_name == DUPE_LIST_SHEET:
                                 processed_data = process_dupe_list_data(values)
-                                new_cache_data["dupes"] = processed_data
+                                new_cache_data["dupes"] = processed_data # Store fetched data temporarily
                                 logger.info(f"Processed {len(processed_data)} items for {DUPE_LIST_SHEET} -> dupes")
                             elif sheet_name in CATEGORIES:
                                 processed_data = process_sheet_data(values)
-                                fetched_values_categories[sheet_name] = processed_data
+                                fetched_values_categories[sheet_name] = processed_data # Store fetched data temporarily
                                 logger.info(f"Processed {len(processed_data)} items for Category: {sheet_name}")
                             else:
                                 logger.warning(f"Unrecognized sheet name '{sheet_name}' derived from range '{range_str}' in Values/Dupes sheet.")
@@ -529,45 +566,10 @@ async def update_cache_periodically():
                             logger.error(f"Error processing data for {sheet_name}: {e}", exc_info=True)
                             current_errors[f"process_{target_key}"] = str(e)
 
-                # --- Detect Value Changes ---
-                logger.info("Comparing fetched values with cached values...")
-                current_time = datetime.now(timezone.utc)
-                detected_value_changes = {}
-                fields_to_compare = ['value', 'dupedValue', 'marketValue']
-
-                if "values" not in cache: cache["values"] = {} # Ensure exists
-
-                for category, new_items in fetched_values_categories.items():
-                    old_items_dict = {item['name']: item for item in cache["values"].get(category, [])}
-                    category_changes = []
-
-                    for new_item in new_items:
-                        item_name = new_item.get('name')
-                        if not item_name or item_name == 'N/A': continue
-
-                        old_item = old_items_dict.get(item_name)
-                        if old_item: # Check existing item for changes
-                            for field in fields_to_compare:
-                                old_val_str = old_item.get(field, 'N/A')
-                                new_val_str = new_item.get(field, 'N/A')
-                                old_norm = parse_cached_currency(old_val_str) if parse_cached_currency(old_val_str) is not None else old_val_str
-                                new_norm = parse_cached_currency(new_val_str) if parse_cached_currency(new_val_str) is not None else new_val_str
-
-                                if old_norm != new_norm:
-                                    logger.info(f"Change detected in {category}: {item_name} - {field}: '{old_val_str}' -> '{new_val_str}'")
-                                    category_changes.append({
-                                        "item_name": item_name, "field": field,
-                                        "old_value": old_val_str if old_val_str is not None else "N/A",
-                                        "new_value": new_val_str if new_val_str is not None else "N/A",
-                                        "timestamp": current_time.isoformat()
-                                    })
-                    if category_changes:
-                        detected_value_changes[category] = category_changes
-
-                # --- Fetch Roblox Avatars ---
-                logger.info("Fetching Roblox avatars...")
+                # --- Fetch Roblox Avatars (for new data before comparison/webhook) ---
+                logger.info("Fetching Roblox avatars for newly processed data...")
                 avatar_tasks = []
-                # Combine lists needing avatars (only user scammers and DWC have roblox usernames)
+                # Combine lists needing avatars from the *newly fetched* data
                 entries_needing_avatars = new_cache_data.get("user_scammers", []) + new_cache_data.get("dwc", [])
                 for entry in entries_needing_avatars:
                      if entry.get('roblox_username'):
@@ -575,7 +577,156 @@ async def update_cache_periodically():
                          avatar_tasks.append(fetch_avatar_for_entry_update(session, entry))
                 if avatar_tasks:
                     await asyncio.gather(*avatar_tasks) # Exceptions logged within helper
-                logger.info(f"Finished fetching avatars for {len(avatar_tasks)} potential entries.")
+                logger.info(f"Finished fetching avatars for {len(avatar_tasks)} potential new entries.")
+
+                # --- Change Detection & Webhook Preparation (BEFORE Cache Update) ---
+                current_time = datetime.now(timezone.utc)
+                timestamp_iso = current_time.isoformat()
+
+                # 1. Value Changes (Existing Logic + Webhook Prep)
+                detected_value_changes_for_api = {} # For the /api/value-changes endpoint
+                if "values" not in cache: cache["values"] = {} # Ensure exists for comparison
+                if VALUE_WEBHOOK_URL and not any(k.startswith("process_values_") for k in current_errors) and "values_dupes_batch" not in current_errors:
+                    logger.info("Detecting value changes for webhooks...")
+                    fields_to_compare = ['value', 'dupedValue', 'marketValue']
+                    for category, new_items in fetched_values_categories.items():
+                        old_items_dict = {item['name']: item for item in cache["values"].get(category, [])}
+                        category_changes_for_api = []
+
+                        for new_item in new_items:
+                            item_name = new_item.get('name')
+                            if not item_name or item_name == 'N/A': continue
+
+                            old_item = old_items_dict.get(item_name)
+                            if old_item: # Check existing item for changes
+                                for field in fields_to_compare:
+                                    old_val_str = old_item.get(field, 'N/A')
+                                    new_val_str = new_item.get(field, 'N/A')
+                                    # Simple string comparison is sufficient here as they are formatted consistently
+                                    if old_val_str != new_val_str:
+                                        logger.info(f"Change detected in {category}: {item_name} - {field}: '{old_val_str}' -> '{new_val_str}'")
+                                        change_info = {
+                                            "item_name": item_name, "field": field,
+                                            "old_value": old_val_str if old_val_str is not None else "N/A",
+                                            "new_value": new_val_str if new_val_str is not None else "N/A",
+                                            "timestamp": timestamp_iso
+                                        }
+                                        category_changes_for_api.append(change_info)
+
+                                        # Prepare webhook embed
+                                        embed = {
+                                            "title": f"Value Update: {item_name} ({category})",
+                                            "color": 3447003, # Blue
+                                            "fields": [
+                                                {"name": "Field Changed", "value": field, "inline": True},
+                                                {"name": "Old Value", "value": f"`{change_info['old_value']}`", "inline": True},
+                                                {"name": "New Value", "value": f"`{change_info['new_value']}`", "inline": True},
+                                                {"name": "Item Notes", "value": new_item.get('notes', 'N/A')[:1020] or 'N/A', "inline": False}, # Limit notes length
+                                            ],
+                                            "timestamp": timestamp_iso
+                                        }
+                                        if new_item.get('icon'):
+                                            embed["thumbnail"] = {"url": new_item['icon']}
+
+                                        webhook_tasks.append(send_webhook_notification(session, VALUE_WEBHOOK_URL, embed))
+                        if category_changes_for_api:
+                            detected_value_changes_for_api[category] = category_changes_for_api
+                    logger.info(f"Prepared {len(webhook_tasks)} value change webhooks.")
+                elif not VALUE_WEBHOOK_URL:
+                    logger.info("VALUE_WEBHOOK_URL not set, skipping value change webhook detection.")
+                else:
+                    logger.warning("Skipping value change webhook detection due to fetch/processing errors.")
+
+
+                # 2. New Scammers / DWC (New Logic + Webhook Prep)
+                if SCAMMER_WEBHOOK_URL and "scammer_dwc_batch" not in current_errors and not any(k.startswith("process_") and k in ["process_user_scammers", "process_server_scammers", "process_dwc"] for k in current_errors):
+                    logger.info("Detecting new scammer/DWC entries for webhooks...")
+                    initial_webhook_task_count = len(webhook_tasks)
+
+                    # User Scammers
+                    old_user_keys = set((item.get('discord_id'), item.get('roblox_username')) for item in cache.get("user_scammers", []))
+                    for item in new_cache_data.get("user_scammers", []):
+                        key = (item.get('discord_id'), item.get('roblox_username'))
+                        if key not in old_user_keys:
+                            logger.info(f"New User Scammer detected: Discord={item.get('discord_id')}, Roblox={item.get('roblox_username')}")
+                            embed = {
+                                "title": "🚨 New User Scammer Added",
+                                "color": 15158332, # Red
+                                "fields": [
+                                    {"name": "Discord ID", "value": f"`{item.get('discord_id', 'N/A')}`", "inline": True},
+                                    {"name": "Roblox User", "value": f"`{item.get('roblox_username', 'N/A')}`", "inline": True},
+                                    {"name": "Scam Type", "value": item.get('scam_type', 'N/A'), "inline": False},
+                                    {"name": "Explanation", "value": item.get('explanation', 'N/A')[:1020] or 'N/A', "inline": False},
+                                ],
+                                "timestamp": timestamp_iso
+                            }
+                            if item.get('evidence_link'):
+                                embed["fields"].append({"name": "Evidence", "value": item['evidence_link'], "inline": False})
+                            if item.get('alt_accounts'):
+                                embed["fields"].append({"name": "Alt Accounts", "value": ", ".join([f"`{a}`" for a in item['alt_accounts']]), "inline": False})
+                            if item.get('roblox_avatar_url'):
+                                embed["thumbnail"] = {"url": item['roblox_avatar_url']}
+                            webhook_tasks.append(send_webhook_notification(session, SCAMMER_WEBHOOK_URL, embed))
+
+                    # Server Scammers
+                    old_server_keys = set((item.get('server_id'), item.get('server_name')) for item in cache.get("server_scammers", []))
+                    for item in new_cache_data.get("server_scammers", []):
+                        key = (item.get('server_id'), item.get('server_name'))
+                        if key not in old_server_keys:
+                            logger.info(f"New Server Scammer detected: ID={item.get('server_id')}, Name={item.get('server_name')}")
+                            embed = {
+                                "title": "🚨 New Server Scammer Added",
+                                "color": 15158332, # Red
+                                "fields": [
+                                    {"name": "Server ID", "value": f"`{item.get('server_id', 'N/A')}`", "inline": True},
+                                    {"name": "Server Name", "value": f"`{item.get('server_name', 'N/A')}`", "inline": True},
+                                    {"name": "Scam Type", "value": item.get('scam_type', 'N/A'), "inline": False},
+                                    {"name": "Explanation", "value": item.get('explanation', 'N/A')[:1020] or 'N/A', "inline": False},
+                                ],
+                                "timestamp": timestamp_iso
+                            }
+                            if item.get('evidence_link'):
+                                embed["fields"].append({"name": "Evidence", "value": item['evidence_link'], "inline": False})
+                            webhook_tasks.append(send_webhook_notification(session, SCAMMER_WEBHOOK_URL, embed))
+
+                    # DWC Entries
+                    old_dwc_keys = set((item.get('discord_user_id'), item.get('discord_server_id'), item.get('roblox_username')) for item in cache.get("dwc", []))
+                    for item in new_cache_data.get("dwc", []):
+                        key = (item.get('discord_user_id'), item.get('discord_server_id'), item.get('roblox_username'))
+                        if key not in old_dwc_keys:
+                            logger.info(f"New DWC Entry detected: User={item.get('discord_user_id')}, Server={item.get('discord_server_id')}, Roblox={item.get('roblox_username')}")
+                            embed = {
+                                "title": "⚠️ New DWC Entry Added",
+                                "color": 15105570, # Orange/Dark Yellow
+                                "fields": [
+                                    {"name": "Discord User ID", "value": f"`{item.get('discord_user_id', 'N/A')}`", "inline": True},
+                                    {"name": "Discord Server ID", "value": f"`{item.get('discord_server_id', 'N/A')}`", "inline": True},
+                                    {"name": "Roblox User", "value": f"`{item.get('roblox_username', 'N/A')}`", "inline": True},
+                                    {"name": "Explanation", "value": item.get('explanation', 'N/A')[:1020] or 'N/A', "inline": False},
+                                ],
+                                "timestamp": timestamp_iso
+                            }
+                            if item.get('evidence_link'):
+                                embed["fields"].append({"name": "Evidence", "value": item['evidence_link'], "inline": False})
+                            if item.get('alt_accounts'):
+                                embed["fields"].append({"name": "Alt Accounts", "value": ", ".join([f"`{a}`" for a in item['alt_accounts']]), "inline": False})
+                            if item.get('roblox_avatar_url'):
+                                embed["thumbnail"] = {"url": item['roblox_avatar_url']}
+                            webhook_tasks.append(send_webhook_notification(session, SCAMMER_WEBHOOK_URL, embed))
+
+                    logger.info(f"Prepared {len(webhook_tasks) - initial_webhook_task_count} new scammer/DWC webhooks.")
+                elif not SCAMMER_WEBHOOK_URL:
+                    logger.info("SCAMMER_WEBHOOK_URL not set, skipping new scammer webhook detection.")
+                else:
+                    logger.warning("Skipping new scammer webhook detection due to fetch/processing errors.")
+
+                # --- Send Webhooks Concurrently ---
+                if webhook_tasks:
+                    logger.info(f"Sending {len(webhook_tasks)} webhook notifications...")
+                    await asyncio.gather(*webhook_tasks)
+                    logger.info("Finished sending webhook notifications.")
+                else:
+                    logger.info("No webhooks to send for this cycle.")
 
 
                 # --- Final Cache Update ---
@@ -587,9 +738,10 @@ async def update_cache_periodically():
                     cache["server_scammers"] = new_cache_data["server_scammers"]
                     cache["dwc"] = new_cache_data["dwc"]
                     cache["dupes"] = new_cache_data["dupes"]
-                    cache["value_changes"] = detected_value_changes
+                    cache["value_changes"] = detected_value_changes_for_api # Store the detected changes
                     cache["last_updated"] = current_time
                     cache["is_ready"] = True
+                    cache["service_available"] = True # Mark as available on success
                     update_occurred = True
                     logger.info(f"Cache update cycle completed successfully.")
                 else: # Errors occurred, attempt partial update
@@ -598,46 +750,60 @@ async def update_cache_periodically():
 
                     # Update values only if the values/dupes batch succeeded AND processing succeeded
                     if "values_dupes_batch" not in current_errors and not any(k.startswith("process_values_") for k in current_errors):
-                        if cache["values"] != fetched_values_categories:
-                            cache["values"] = fetched_values_categories
-                            cache["value_changes"] = detected_value_changes # Update changes along with values
-                            partial_update_details.append("values")
-                            update_occurred = True
+                        # Check if fetched data is different from cache before updating
+                        if cache.get("values") != fetched_values_categories:
+                             cache["values"] = fetched_values_categories
+                             cache["value_changes"] = detected_value_changes_for_api # Update changes along with values
+                             partial_update_details.append("values")
+                             update_occurred = True
+                    else:
+                         logger.warning("Skipping update for 'values' due to errors.")
+
 
                     # Update dupes only if the values/dupes batch succeeded AND processing succeeded
                     if "values_dupes_batch" not in current_errors and "process_dupes" not in current_errors:
-                         if cache["dupes"] != new_cache_data["dupes"]:
+                         if cache.get("dupes") != new_cache_data["dupes"]:
                             cache["dupes"] = new_cache_data["dupes"]
                             partial_update_details.append("dupes")
                             update_occurred = True
+                    else:
+                         logger.warning("Skipping update for 'dupes' due to errors.")
 
                     # Update scammer/DWC sections if their batch succeeded AND processing succeeded
                     if "scammer_dwc_batch" not in current_errors:
                         for key in ["user_scammers", "server_scammers", "dwc"]:
                             process_error_key = f"process_{key}"
                             if process_error_key not in current_errors:
-                                 if cache[key] != new_cache_data[key]:
+                                 if cache.get(key) != new_cache_data[key]:
                                     cache[key] = new_cache_data[key]
                                     partial_update_details.append(key)
                                     update_occurred = True
+                            else:
+                                logger.warning(f"Skipping update for '{key}' due to processing error.")
+                    else:
+                         logger.warning("Skipping update for 'user_scammers', 'server_scammers', 'dwc' due to batch fetch error.")
+
 
                     if update_occurred:
                          cache["last_updated"] = current_time # Mark partial update time
                          cache["is_ready"] = True # Allow access even if partial
+                         # Keep service_available as potentially false if there were fetch errors
                          logger.info(f"Partially updated cache sections: {', '.join(partial_update_details)}")
                     else:
                          logger.error(f"Cache update cycle failed, and no parts could be updated based on errors. Errors: {current_errors}")
-                         # Keep cache["is_ready"] as it was.
+                         # Keep cache["is_ready"] as it was. Don't update timestamp.
 
             except Exception as e:
                 logger.exception(f"Critical error during cache update cycle: {e}")
+                # If a critical error happens (e.g., network error during fetch), mark service potentially unavailable
                 if isinstance(e, (aiohttp.ClientError, HttpError, asyncio.TimeoutError)):
                     logger.warning("Communication error detected, will re-check service availability next cycle.")
+                    # service_available might have already been set to False by fetch_batch_ranges_async
 
             # --- Wait for the next cycle ---
             end_time = datetime.now(timezone.utc)
             duration = (end_time - start_time).total_seconds()
-            wait_time = max(10, CACHE_UPDATE_INTERVAL_SECONDS - duration)
+            wait_time = max(10, CACHE_UPDATE_INTERVAL_SECONDS - duration) # Ensure at least 10s wait
             logger.info(f"Cache update cycle duration: {duration:.2f}s. Waiting {wait_time:.2f}s for next cycle.")
             await asyncio.sleep(wait_time)
 
@@ -647,9 +813,7 @@ async def fetch_avatar_for_entry_update(session: aiohttp.ClientSession, entry: d
     roblox_username = entry.get('roblox_username')
     if not roblox_username: return
 
-    current_avatar = entry.get('roblox_avatar_url')
     new_avatar = None # Default to None
-
     try:
         user_id = await get_roblox_user_id(session, roblox_username)
         if user_id:
@@ -661,9 +825,8 @@ async def fetch_avatar_for_entry_update(session: aiohttp.ClientSession, entry: d
         # Keep new_avatar as None on error
 
     finally:
-         # Update the dict only if the value has actually changed
-         if current_avatar != new_avatar:
-             entry['roblox_avatar_url'] = new_avatar
+         # Update the entry dict directly (no need to check if changed, just set it)
+         entry['roblox_avatar_url'] = new_avatar
 
 
 # --- FastAPI Startup Event ---
@@ -673,15 +836,27 @@ async def startup_event():
     if not cache["service_available"]:
          logger.warning("Google Sheets service not available at startup. Will attempt re-init in background task.")
     logger.info("Starting background cache update task...")
+    # Check for webhook URLs at startup
+    if not SCAMMER_WEBHOOK_URL:
+        logger.warning("SCAMMER_WEBHOOK_URL environment variable not set. New scammer notifications disabled.")
+    if not VALUE_WEBHOOK_URL:
+        logger.warning("VALUE_WEBHOOK_URL environment variable not set. Value change notifications disabled.")
     asyncio.create_task(update_cache_periodically())
 
 
-# --- API Endpoints (Largely unchanged, rely on cache state) ---
+# --- API Endpoints ---
 
 def check_cache_readiness():
     """Reusable check for API endpoints - Checks cache readiness"""
     if not cache["is_ready"]:
-         raise HTTPException(status_code=503, detail="Cache is initializing or data is currently unavailable. Please try again shortly.")
+         # Be more specific if service is known to be down
+         if not cache["service_available"]:
+              raise HTTPException(status_code=503, detail="Service temporarily unavailable due to backend connection issues. Please try again later.")
+         else:
+              raise HTTPException(status_code=503, detail="Cache is initializing or data is currently unavailable. Please try again shortly.")
+    # Optional: Add check for staleness?
+    # if cache["last_updated"] and (datetime.now(timezone.utc) - cache["last_updated"]).total_seconds() > CACHE_UPDATE_INTERVAL_SECONDS * 3:
+    #      raise HTTPException(status_code=503, detail="Data may be stale. Update in progress or backend issue.")
 
 @app.get("/")
 async def root():
@@ -690,47 +865,59 @@ async def root():
 @app.get("/api/status")
 async def get_status():
     """Returns the current status of the cache and service availability"""
+    last_updated_iso = cache["last_updated"].isoformat() if cache["last_updated"] else None
     return {
         "cache_ready": cache["is_ready"],
         "sheets_service_available": cache["service_available"],
-        "last_updated": cache["last_updated"].isoformat() if cache["last_updated"] else None,
+        "last_updated": last_updated_iso,
         "cached_items": {
-            "value_categories": len(cache["values"]),
-            "user_scammers": len(cache["user_scammers"]),
-            "server_scammers": len(cache["server_scammers"]),
-            "dwc_entries": len(cache["dwc"]),
-            "duped_usernames": len(cache["dupes"]),
+            "value_categories": len(cache.get("values", {})),
+            "user_scammers": len(cache.get("user_scammers", [])),
+            "server_scammers": len(cache.get("server_scammers", [])),
+            "dwc_entries": len(cache.get("dwc", [])),
+            "duped_usernames": len(cache.get("dupes", [])),
         },
-         "value_change_categories": len(cache.get("value_changes", {}))
+         "value_change_categories_in_last_cycle": len(cache.get("value_changes", {}))
     }
 
 @app.get("/api/values")
 async def get_values():
     """Get all values data from cache"""
     check_cache_readiness()
-    return cache["values"]
+    return cache.get("values", {})
 
 @app.get("/api/values/{category}")
 async def get_category_values(category: str):
     """Get values data for a specific category from cache"""
     check_cache_readiness()
-    matched_category = next((c for c in CATEGORIES if c.lower() == category.lower()), None)
+    # Case-insensitive matching for category name
+    matched_category = next((c for c in cache.get("values", {}).keys() if c.lower() == category.lower()), None)
     if not matched_category:
-         raise HTTPException(status_code=404, detail=f"Category '{category}' not found.")
-    return {matched_category: cache["values"].get(matched_category, [])}
+         # Check if the category *exists* conceptually even if empty
+         if category.lower() in [c.lower() for c in CATEGORIES]:
+              return {category: []} # Return empty list if category is valid but has no items
+         else:
+            raise HTTPException(status_code=404, detail=f"Category '{category}' not found.")
+    return {matched_category: cache.get("values", {}).get(matched_category, [])}
+
 
 @app.get("/api/value-changes/{category}")
 async def get_category_value_changes(category: str):
-    """Get detected value changes for a specific category."""
+    """Get detected value changes for a specific category from the last cache update cycle."""
     check_cache_readiness()
-    matched_category = next((c for c in CATEGORIES if c.lower() == category.lower()), None)
+    # Case-insensitive matching for category name
+    matched_category = next((c for c in cache.get("value_changes", {}).keys() if c.lower() == category.lower()), None)
     if not matched_category:
-         raise HTTPException(status_code=404, detail=f"Category '{category}' not found.")
+         # Check if the category *exists* conceptually even if empty
+         if category.lower() in [c.lower() for c in CATEGORIES]:
+              return {category: []} # Return empty list if category is valid but had no changes
+         else:
+            raise HTTPException(status_code=404, detail=f"Category '{category}' not found.")
     return {matched_category: cache.get("value_changes", {}).get(matched_category, [])}
 
 @app.get("/api/value-changes")
 async def get_all_value_changes():
-    """Get all detected value changes from the last cycle."""
+    """Get all detected value changes from the last cache update cycle."""
     check_cache_readiness()
     return cache.get("value_changes", {})
 
@@ -739,17 +926,16 @@ async def get_scammers():
     """Get all scammer and DWC data (users, servers, dwc) from cache"""
     check_cache_readiness()
     return {
-        "users": cache["user_scammers"],
-        "servers": cache["server_scammers"],
-        "dwc": cache["dwc"]
+        "users": cache.get("user_scammers", []),
+        "servers": cache.get("server_scammers", []),
+        "dwc": cache.get("dwc", [])
     }
 
 @app.get("/api/dupes")
 async def get_dupes():
     """Get all duped usernames from cache"""
     check_cache_readiness()
-    # Handle case where dupes might be None temporarily during init failure
-    return {"usernames": cache.get("dupes") or []}
+    return {"usernames": cache.get("dupes", [])} # Return empty list if not ready or empty
 
 
 class UsernameCheck(BaseModel):
@@ -761,32 +947,33 @@ async def check_username(data: UsernameCheck):
     check_cache_readiness() # Use the standard readiness check
 
     username_to_check = data.username.strip().lower()
-    is_duped = username_to_check in (cache.get("dupes") or [])
+    # Ensure dupes list is populated before checking
+    dupes_list = cache.get("dupes", [])
+    is_duped = username_to_check in dupes_list
 
-    # Webhook notification (runs in background)
+    # Webhook notification for checks resulting in "Not Found" (runs in background)
+    # Note: This uses the *value* webhook URL per original code, not the scammer one.
+    # If this needs to go to a *different* webhook, adjust the env var name.
     if not is_duped:
-        webhook_url = os.getenv("WEBHOOK_URL")
+        webhook_url = os.getenv("WEBHOOK_URL") # Keep original env var name for this specific check
         if webhook_url:
-            async def send_webhook_notification():
-                try:
-                    async with aiohttp.ClientSession() as session:
-                        webhook_data = {
-                            "content": None,
-                            "embeds": [{
-                                "title": "New Dupe Check - Not Found",
-                                "description": f"Username `{data.username}` was checked but not found in the dupe database.",
+            # Use a new session for this one-off task or pass the main one if safe
+            async def send_check_webhook():
+                 try:
+                    async with aiohttp.ClientSession() as session: # Create a short-lived session
+                        embed = {
+                                "title": "User Dupe Check - Not Found",
+                                "description": f"Username `{data.username}` was checked against the dupe list but was **not** found.",
                                 "color": 16776960, # Yellow
                                 "timestamp": datetime.now(timezone.utc).isoformat()
-                            }]
                         }
-                        async with session.post(webhook_url, json=webhook_data) as response:
-                            if response.status not in [200, 204]:
-                                logger.warning(f"Failed to send webhook (Status: {response.status}): {await response.text()}")
-                except Exception as e:
-                    logger.error(f"Error sending webhook: {e}")
-            asyncio.create_task(send_webhook_notification())
+                        await send_webhook_notification(session, webhook_url, embed)
+                 except Exception as e:
+                     logger.error(f"Error sending dupe check webhook: {e}") # Log errors from the task
+
+            asyncio.create_task(send_check_webhook()) # Fire and forget
         else:
-            logger.info("Webhook URL not configured. Skipping notification.")
+            logger.info("WEBHOOK_URL (for dupe checks) not configured. Skipping notification.")
 
     return {"username": data.username, "is_duped": is_duped}
 
@@ -794,12 +981,34 @@ async def check_username(data: UsernameCheck):
 @app.get("/health")
 def health_check():
     """Provides a health status of the API and its cache."""
+    now = datetime.now(timezone.utc)
+    status_detail = {"status": "ok", "last_updated": None, "time_since_update_seconds": None}
+
+    if cache["last_updated"]:
+        status_detail["last_updated"] = cache["last_updated"].isoformat()
+        time_since_update = (now - cache["last_updated"]).total_seconds()
+        status_detail["time_since_update_seconds"] = round(time_since_update)
+    else:
+        status_detail["last_updated"] = None
+        status_detail["time_since_update_seconds"] = None
+
+
     if not cache["is_ready"]:
-        return {"status": "initializing"}
-    if not cache["service_available"]:
-        return {"status": "degraded", "reason": "Sheets service connection issue"}
-    if cache["last_updated"] and (datetime.now(timezone.utc) - cache["last_updated"]).total_seconds() > CACHE_UPDATE_INTERVAL_SECONDS * 3:
-         return {"status": "degraded", "reason": "Cache potentially stale (last update > 3 intervals ago)"}
-    return {"status": "ok"}
+        status_detail["status"] = "initializing"
+        status_detail["reason"] = "Cache has not been populated yet."
+        return status_detail
 
-# --- END OF FILE main.py ---
+    if not cache["service_available"]:
+        status_detail["status"] = "degraded"
+        status_detail["reason"] = "Google Sheets service connection issue detected on last attempt."
+        return status_detail
+
+    # Check for staleness only if the cache is ready and service *was* available last check
+    # Allow some grace period (e.g., 3 intervals)
+    if cache["last_updated"] and time_since_update > CACHE_UPDATE_INTERVAL_SECONDS * 3:
+         status_detail["status"] = "degraded"
+         status_detail["reason"] = f"Cache potentially stale (last update > {CACHE_UPDATE_INTERVAL_SECONDS * 3} seconds ago)"
+         return status_detail
+
+    # If we reach here, status is 'ok'
+    return status_detail