Spaces:
Sleeping
Sleeping
| """ | |
| Normalize raw scraped event/attraction data → events_calendar.json schema. | |
| Merges into existing events_calendar.json (append, deduped by name). | |
| """ | |
| import json | |
| import logging | |
| import re | |
| from typing import Optional | |
| try: | |
| from rapidfuzz import fuzz | |
| HAS_FUZZ = True | |
| except ImportError: | |
| HAS_FUZZ = False | |
| from scripts.scrapers.config import PATHS, SCRAPER_SETTINGS | |
| logger = logging.getLogger(__name__) | |
| # Known recurring annual festivals mapped by keyword | |
| KNOWN_FESTIVALS = { | |
| "new year": {"month": 1, "day": 1, "duration_days": 1, "impact": "high", "recurring": "yearly"}, | |
| "lunar new year": {"month": 1, "day": 29, "duration_days": 7, "impact": "high", "recurring": "yearly"}, | |
| "christmas": {"month": 12, "day": 25, "duration_days": 3, "impact": "high", "recurring": "yearly"}, | |
| "halloween": {"month": 10, "day": 31, "duration_days": 1, "impact": "medium", "recurring": "yearly"}, | |
| "carnival": {"month": 2, "day": 1, "duration_days": 5, "impact": "high", "recurring": "yearly"}, | |
| "oktoberfest": {"month": 9, "day": 16, "duration_days": 18, "impact": "high", "recurring": "yearly"}, | |
| "cherry blossom": {"month": 4, "day": 1, "duration_days": 14, "impact": "high", "recurring": "yearly"}, | |
| "songkran": {"month": 4, "day": 13, "duration_days": 3, "impact": "high", "recurring": "yearly"}, | |
| "diwali": {"month": 11, "day": 1, "duration_days": 5, "impact": "high", "recurring": "yearly"}, | |
| "ramadan": {"month": 3, "day": 10, "duration_days": 30, "impact": "medium", "recurring": "yearly"}, | |
| "eid": {"month": 4, "day": 10, "duration_days": 3, "impact": "medium", "recurring": "yearly"}, | |
| "holi": {"month": 3, "day": 25, "duration_days": 2, "impact": "high", "recurring": "yearly"}, | |
| "lantern festival": {"month": 2, "day": 15, "duration_days": 1, "impact": "medium", "recurring": "yearly"}, | |
| "bastille": {"month": 7, "day": 14, "duration_days": 1, "impact": "medium", "recurring": "yearly"}, | |
| "marathon": {"month": 11, "day": 1, "duration_days": 1, "impact": "low", "recurring": "yearly"}, | |
| "film festival": {"month": 9, "day": 1, "duration_days": 10, "impact": "medium", "recurring": "yearly"}, | |
| "fashion week": {"month": 9, "day": 1, "duration_days": 7, "impact": "medium", "recurring": "yearly"}, | |
| "fireworks": {"month": 7, "day": 4, "duration_days": 1, "impact": "medium", "recurring": "yearly"}, | |
| "music festival": {"month": 7, "day": 1, "duration_days": 3, "impact": "medium", "recurring": "yearly"}, | |
| "pride": {"month": 6, "day": 15, "duration_days": 7, "impact": "medium", "recurring": "yearly"}, | |
| "tet": {"month": 1, "day": 29, "duration_days": 7, "impact": "high", "recurring": "yearly"}, | |
| "full moon": {"month": 0, "day": 15, "duration_days": 1, "impact": "low", "recurring": "monthly"}, | |
| } | |
| def _classify_impact(reviews_count: int) -> str: | |
| if reviews_count >= 1000: | |
| return "high" | |
| if reviews_count >= 100: | |
| return "medium" | |
| return "low" | |
| def _match_known_festival(name: str) -> Optional[dict]: | |
| """Match event name against known festivals.""" | |
| name_lower = name.lower() | |
| for keyword, meta in KNOWN_FESTIVALS.items(): | |
| if keyword in name_lower: | |
| return meta | |
| return None | |
| def normalize_event(raw: dict) -> Optional[dict]: | |
| """ | |
| Convert raw scraped event dict to events_calendar.json schema. | |
| Returns None if entry lacks minimum required fields. | |
| """ | |
| name = (raw.get("name") or "").strip() | |
| if not name or len(name) < 3: | |
| return None | |
| # Try to match known festival patterns | |
| meta = _match_known_festival(name) | |
| month = raw.get("month") or (meta["month"] if meta else 0) | |
| day = raw.get("day") or (meta["day"] if meta else 1) | |
| duration = raw.get("duration_days") or (meta["duration_days"] if meta else 1) | |
| impact = _classify_impact(raw.get("reviews_count", 0)) | |
| if meta: | |
| impact = meta.get("impact", impact) | |
| result = { | |
| "name_vi": "", | |
| "name_en": (raw.get("name_en") or name).strip(), | |
| "month": int(month), | |
| "day": int(day), | |
| "duration_days": int(duration), | |
| "description_vi": "", | |
| "description_en": (raw.get("description") or "")[:400].strip(), | |
| "destinations": [raw["destination_id"]] if raw.get("destination_id") else [], | |
| "impact": impact, | |
| "_source": raw.get("source", "unknown"), | |
| } | |
| # Add recurring flag if detected | |
| if meta and meta.get("recurring"): | |
| result["recurring"] = meta["recurring"] | |
| if meta.get("recurring") == "monthly" and meta.get("day"): | |
| result["recurring_day"] = meta["day"] | |
| return result | |
| def is_duplicate_event(new: dict, existing: list) -> bool: | |
| """Check if an event is a duplicate by name similarity.""" | |
| new_name = new["name_en"].lower().strip() | |
| threshold = SCRAPER_SETTINGS["dedup_name_threshold"] | |
| for ex in existing: | |
| ex_name = (ex.get("name_en") or ex.get("name_vi", "")).lower().strip() | |
| if HAS_FUZZ: | |
| if fuzz.ratio(new_name, ex_name) >= threshold: | |
| return True | |
| else: | |
| if new_name == ex_name: | |
| return True | |
| return False | |
| def merge_into_events_db(new_events: list) -> dict: | |
| """ | |
| Load existing events_calendar.json, append new events (deduped), save. | |
| """ | |
| events_db_path = PATHS["events_db"] | |
| with open(events_db_path, "r", encoding="utf-8") as f: | |
| db = json.load(f) | |
| existing = db.get("events", []) | |
| added = 0 | |
| skipped_invalid = 0 | |
| skipped_dup = 0 | |
| for raw in new_events: | |
| normalized = normalize_event(raw) | |
| if normalized is None: | |
| skipped_invalid += 1 | |
| continue | |
| if is_duplicate_event(normalized, existing): | |
| # If same event, enrich destinations list | |
| for ex in existing: | |
| ex_name = (ex.get("name_en") or "").lower() | |
| if ex_name == normalized["name_en"].lower(): | |
| dest = normalized["destinations"][0] if normalized["destinations"] else None | |
| if dest and dest not in ex.get("destinations", []): | |
| ex.setdefault("destinations", []).append(dest) | |
| skipped_dup += 1 | |
| continue | |
| existing.append(normalized) | |
| added += 1 | |
| db["events"] = existing | |
| with open(events_db_path, "w", encoding="utf-8") as f: | |
| json.dump(db, f, ensure_ascii=False, indent=2) | |
| stats = { | |
| "total_in_db": len(existing), | |
| "added": added, | |
| "skipped_invalid": skipped_invalid, | |
| "skipped_duplicate": skipped_dup, | |
| } | |
| logger.info(f"Event merge: +{added} added, {skipped_dup} dupes → {len(existing)} total") | |
| return stats | |