""" Normalize raw scraped event/attraction data → events_calendar.json schema. Merges into existing events_calendar.json (append, deduped by name). """ import json import logging import re from typing import Optional try: from rapidfuzz import fuzz HAS_FUZZ = True except ImportError: HAS_FUZZ = False from scripts.scrapers.config import PATHS, SCRAPER_SETTINGS logger = logging.getLogger(__name__) # Known recurring annual festivals mapped by keyword KNOWN_FESTIVALS = { "new year": {"month": 1, "day": 1, "duration_days": 1, "impact": "high", "recurring": "yearly"}, "lunar new year": {"month": 1, "day": 29, "duration_days": 7, "impact": "high", "recurring": "yearly"}, "christmas": {"month": 12, "day": 25, "duration_days": 3, "impact": "high", "recurring": "yearly"}, "halloween": {"month": 10, "day": 31, "duration_days": 1, "impact": "medium", "recurring": "yearly"}, "carnival": {"month": 2, "day": 1, "duration_days": 5, "impact": "high", "recurring": "yearly"}, "oktoberfest": {"month": 9, "day": 16, "duration_days": 18, "impact": "high", "recurring": "yearly"}, "cherry blossom": {"month": 4, "day": 1, "duration_days": 14, "impact": "high", "recurring": "yearly"}, "songkran": {"month": 4, "day": 13, "duration_days": 3, "impact": "high", "recurring": "yearly"}, "diwali": {"month": 11, "day": 1, "duration_days": 5, "impact": "high", "recurring": "yearly"}, "ramadan": {"month": 3, "day": 10, "duration_days": 30, "impact": "medium", "recurring": "yearly"}, "eid": {"month": 4, "day": 10, "duration_days": 3, "impact": "medium", "recurring": "yearly"}, "holi": {"month": 3, "day": 25, "duration_days": 2, "impact": "high", "recurring": "yearly"}, "lantern festival": {"month": 2, "day": 15, "duration_days": 1, "impact": "medium", "recurring": "yearly"}, "bastille": {"month": 7, "day": 14, "duration_days": 1, "impact": "medium", "recurring": "yearly"}, "marathon": {"month": 11, "day": 1, "duration_days": 1, "impact": "low", "recurring": "yearly"}, "film festival": {"month": 9, "day": 1, "duration_days": 10, "impact": "medium", "recurring": "yearly"}, "fashion week": {"month": 9, "day": 1, "duration_days": 7, "impact": "medium", "recurring": "yearly"}, "fireworks": {"month": 7, "day": 4, "duration_days": 1, "impact": "medium", "recurring": "yearly"}, "music festival": {"month": 7, "day": 1, "duration_days": 3, "impact": "medium", "recurring": "yearly"}, "pride": {"month": 6, "day": 15, "duration_days": 7, "impact": "medium", "recurring": "yearly"}, "tet": {"month": 1, "day": 29, "duration_days": 7, "impact": "high", "recurring": "yearly"}, "full moon": {"month": 0, "day": 15, "duration_days": 1, "impact": "low", "recurring": "monthly"}, } def _classify_impact(reviews_count: int) -> str: if reviews_count >= 1000: return "high" if reviews_count >= 100: return "medium" return "low" def _match_known_festival(name: str) -> Optional[dict]: """Match event name against known festivals.""" name_lower = name.lower() for keyword, meta in KNOWN_FESTIVALS.items(): if keyword in name_lower: return meta return None def normalize_event(raw: dict) -> Optional[dict]: """ Convert raw scraped event dict to events_calendar.json schema. Returns None if entry lacks minimum required fields. """ name = (raw.get("name") or "").strip() if not name or len(name) < 3: return None # Try to match known festival patterns meta = _match_known_festival(name) month = raw.get("month") or (meta["month"] if meta else 0) day = raw.get("day") or (meta["day"] if meta else 1) duration = raw.get("duration_days") or (meta["duration_days"] if meta else 1) impact = _classify_impact(raw.get("reviews_count", 0)) if meta: impact = meta.get("impact", impact) result = { "name_vi": "", "name_en": (raw.get("name_en") or name).strip(), "month": int(month), "day": int(day), "duration_days": int(duration), "description_vi": "", "description_en": (raw.get("description") or "")[:400].strip(), "destinations": [raw["destination_id"]] if raw.get("destination_id") else [], "impact": impact, "_source": raw.get("source", "unknown"), } # Add recurring flag if detected if meta and meta.get("recurring"): result["recurring"] = meta["recurring"] if meta.get("recurring") == "monthly" and meta.get("day"): result["recurring_day"] = meta["day"] return result def is_duplicate_event(new: dict, existing: list) -> bool: """Check if an event is a duplicate by name similarity.""" new_name = new["name_en"].lower().strip() threshold = SCRAPER_SETTINGS["dedup_name_threshold"] for ex in existing: ex_name = (ex.get("name_en") or ex.get("name_vi", "")).lower().strip() if HAS_FUZZ: if fuzz.ratio(new_name, ex_name) >= threshold: return True else: if new_name == ex_name: return True return False def merge_into_events_db(new_events: list) -> dict: """ Load existing events_calendar.json, append new events (deduped), save. """ events_db_path = PATHS["events_db"] with open(events_db_path, "r", encoding="utf-8") as f: db = json.load(f) existing = db.get("events", []) added = 0 skipped_invalid = 0 skipped_dup = 0 for raw in new_events: normalized = normalize_event(raw) if normalized is None: skipped_invalid += 1 continue if is_duplicate_event(normalized, existing): # If same event, enrich destinations list for ex in existing: ex_name = (ex.get("name_en") or "").lower() if ex_name == normalized["name_en"].lower(): dest = normalized["destinations"][0] if normalized["destinations"] else None if dest and dest not in ex.get("destinations", []): ex.setdefault("destinations", []).append(dest) skipped_dup += 1 continue existing.append(normalized) added += 1 db["events"] = existing with open(events_db_path, "w", encoding="utf-8") as f: json.dump(db, f, ensure_ascii=False, indent=2) stats = { "total_in_db": len(existing), "added": added, "skipped_invalid": skipped_invalid, "skipped_duplicate": skipped_dup, } logger.info(f"Event merge: +{added} added, {skipped_dup} dupes → {len(existing)} total") return stats