""" Normalize scraped hotel data → backend hotels.json seeder schema. Outputs to scraped_data/hotels_international.json (separate from VN hotels). """ import json import logging import re import os from typing import Optional from scripts.scrapers.config import PATHS logger = logging.getLogger(__name__) # Default vendor ID for scraped hotels (system vendor) DEFAULT_VENDOR_ID = "000000000000000000000001" HOTEL_AMENITY_NORMALIZE = { "wifi": "Free WiFi", "wi-fi": "Free WiFi", "internet": "Free WiFi", "pool": "Swimming Pool", "swimming": "Swimming Pool", "gym": "Gym", "fitness": "Gym", "spa": "Spa", "restaurant": "Restaurant", "bar": "Bar", "parking": "Parking", "airport": "Airport Shuttle", "shuttle": "Airport Shuttle", "breakfast": "Breakfast Included", "air conditioning": "Air Conditioning", "ac": "Air Conditioning", "front desk": "24-hour Front Desk", "24-hour": "24-hour Front Desk", "laundry": "Laundry Service", "concierge": "Concierge", "business": "Business Center", "meeting": "Meeting Rooms", "room service": "Room Service", } def _slugify(text: str) -> str: text = text.lower().strip() text = re.sub(r"[^\w\s-]", "", text) text = re.sub(r"[\s_-]+", "-", text) return text[:80] def _normalize_amenities(raw_amenities: list) -> list: """Normalize amenity strings to standardized names.""" normalized = set() for amenity in raw_amenities: if not amenity: continue lower = amenity.lower() matched = False for keyword, standard in HOTEL_AMENITY_NORMALIZE.items(): if keyword in lower: normalized.add(standard) matched = True break if not matched and len(amenity) > 2: normalized.add(amenity.title()) return sorted(list(normalized))[:15] # Cap at 15 amenities def _convert_price_to_vnd(price_usd: Optional[float], exchange_rate: float = 25000) -> int: """Convert USD price/night to VND.""" if not price_usd: return 1_000_000 # Default ~$40/night in VND return int(price_usd * exchange_rate) def normalize_hotel(raw: dict, index: int) -> Optional[dict]: """ Convert raw scraped hotel dict to backend hotels.json schema entry. Returns None if minimum required fields are missing. """ name = (raw.get("name") or "").strip() if not name or len(name) < 3: return None dest_id = raw.get("destination_id", "unknown") city = raw.get("city", "") country = raw.get("country", "") exchange_rate = raw.get("exchange_rate", 25000) lat = raw.get("lat") lon = raw.get("lon") if lat is None or lon is None: # Skip hotels without coordinates (can't display on map) return None star_rating = raw.get("star_rating") or 3 star_rating = min(max(int(star_rating), 1), 5) rating = raw.get("rating") if rating is not None: try: rating = float(rating) if rating > 5: rating = round(rating / 2, 1) rating = round(min(max(rating, 1.0), 5.0), 1) except (TypeError, ValueError): rating = 3.5 else: rating = 3.5 price_usd = raw.get("price_per_night") lowest_price_vnd = _convert_price_to_vnd(price_usd, exchange_rate) amenities = _normalize_amenities(raw.get("amenities") or []) images = raw.get("images") or [] if not images: images = [] # No placeholder images hotel_type = raw.get("property_type") or "HOTEL" # Build locationId reference (will be created as new location) location_id = f"loc_{_slugify(city)}_001" return { "hotelID": f"hotel_{dest_id}_{index:03d}", "vendorId": DEFAULT_VENDOR_ID, "locationId": location_id, "city": city, "country": country, "name": name, "slug": _slugify(name), "hotelType": hotel_type, "starRating": star_rating, "address": (raw.get("address") or f"{city}, {country}").strip(), "latitude": float(lat), "longitude": float(lon), "description": (raw.get("description") or f"Welcome to {name} in {city}.").strip(), "shortDescription": f"{star_rating}-star {hotel_type.lower().replace('_', ' ')} in {city}", "phone": raw.get("phone") or "", "email": "", "website": raw.get("website") or "", "images": images[:5], "amenities": amenities, "policies": { "cancellation": "Free cancellation up to 24 hours before check-in", "pets": False, "smoking": False, }, "approvalStatus": "APPROVED", "status": "ACTIVE", "featured": rating >= 4.0 and star_rating >= 4, "verified": True, "averageRating": rating, "totalReviews": raw.get("reviews_count") or 0, "totalRooms": 50, "lowestPrice": lowest_price_vnd, "createdAt": "2026-01-01T00:00:00Z", "updatedAt": "2026-01-01T00:00:00Z", "_source": raw.get("source", "unknown"), } def save_hotels(scraped_cities: list) -> dict: """ Process all scraped hotel data, normalize, deduplicate by name+city, and save to hotels_international.json. """ output_path = PATHS["hotels_backend"] os.makedirs(os.path.dirname(output_path), exist_ok=True) # Load existing if any existing = [] if os.path.exists(output_path): with open(output_path, "r", encoding="utf-8") as f: existing = json.load(f) existing_keys = {(h["name"].lower(), h.get("city", "").lower()) for h in existing} added = 0 skipped = 0 index_start = len(existing) for city_data in scraped_cities: raw_hotels = city_data.get("hotels", []) city_name = city_data.get("city", "") for i, raw in enumerate(raw_hotels): key = (raw.get("name", "").lower(), city_name.lower()) if key in existing_keys: skipped += 1 continue normalized = normalize_hotel(raw, index_start + added) if normalized is None: skipped += 1 continue existing.append(normalized) existing_keys.add(key) added += 1 with open(output_path, "w", encoding="utf-8") as f: json.dump(existing, f, ensure_ascii=False, indent=2) stats = { "total_hotels": len(existing), "added": added, "skipped": skipped, "output_file": output_path, } logger.info(f"Hotel save: +{added} added, {skipped} skipped → {len(existing)} total → {output_path}") return stats