scripts/collect_real_intent_data.py

"""
Collect & Merge Real Intent Data
==================================
Orchestrates all scrapers and merges results into intent_train.json.

Steps:
  1. Run HuggingFace dataset scraper  (multiwoz, sgd, nlu corpora)
  2. Run Reddit scraper               (r/travel, r/solotravel, ...)
  3. Run TripAdvisor Forum scraper    (forum thread titles)
  4. Merge all scraped files + existing intent_train.json
  5. Deduplicate by text (case-insensitive)
  6. Cap per-intent at TARGET_CAP to avoid new imbalance
  7. Write merged result back to intent_train.json

IMPORTANT: This script never writes synthetic/hardcoded text.
           All samples come from real scraped or published datasets.

Run from chatbot-ml-service/ directory:
    python scripts/collect_real_intent_data.py [--skip-scrape] [--dry-run]

Flags:
  --skip-scrape   : skip running scrapers (use already-saved scraped_*.json files)
  --dry-run       : show counts only, do not write to intent_train.json
"""

import argparse
import json
import logging
import os
import subprocess
import sys
from collections import Counter
from typing import Optional

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
logger = logging.getLogger(__name__)

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATASETS_DIR = os.path.join(BASE_DIR, "app", "data", "datasets")

INTENT_TRAIN_FILE = os.path.join(DATASETS_DIR, "intent_train.json")

SCRAPED_FILES = [
    os.path.join(DATASETS_DIR, "scraped_hf_intent.json"),
    os.path.join(DATASETS_DIR, "scraped_reddit_intent.json"),
    os.path.join(DATASETS_DIR, "scraped_tripadvisor_intent.json"),
]

SCRAPER_SCRIPTS = [
    os.path.join(BASE_DIR, "scripts", "scrapers", "huggingface_dataset_scraper.py"),
    os.path.join(BASE_DIR, "scripts", "scrapers", "reddit_scraper.py"),
    os.path.join(BASE_DIR, "scripts", "scrapers", "tripadvisor_qa_scraper.py"),
]

# Known valid intents in this project
ALLOWED_INTENTS = {
    "weather_info", "budget_advice", "find_flight", "fallback",
    "food_recommend", "transport_info", "plan_trip", "activity_suggest",
    "visa_info", "greeting", "find_hotel", "thank",
    "compare_destinations", "show_map", "destination_info",
    "nearby_attractions", "travel_tips",
}

# Per-intent target cap — we don't want to over-weight any single source
TARGET_CAP = 500
# Underrepresented intents get priority — don't cap them as strictly
PRIORITY_INTENTS = {"destination_info", "show_map", "compare_destinations",
                    "nearby_attractions", "travel_tips", "visa_info",
                    "find_hotel", "greeting", "thank"}
PRIORITY_CAP = 500  # same cap but filled first


def load_json(path: str) -> list[dict]:
    if not os.path.exists(path):
        logger.warning(f"File not found: {path}")
        return []
    with open(path, encoding="utf-8") as f:
        try:
            return json.load(f)
        except json.JSONDecodeError as e:
            logger.error(f"JSON error in {path}: {e}")
            return []


def run_scraper(script: str) -> bool:
    logger.info(f"Running scraper: {script}")
    result = subprocess.run(
        [sys.executable, script],
        cwd=BASE_DIR,
        capture_output=False,
    )
    if result.returncode != 0:
        logger.error(f"Scraper failed: {script}")
        return False
    return True


def merge_and_cap(
    existing: list[dict],
    scraped: list[dict],
    target_cap: int,
    priority_cap: int,
) -> list[dict]:
    """
    Merge existing + scraped samples.
    Priority intents are filled first from scraped data.
    Then existing data fills remaining slots.
    Then non-priority scraped data fills remaining slots.
    Per-intent caps are respected throughout.
    """

    # Start with existing data (ground truth baseline)
    counts: Counter = Counter(s["intent"] for s in existing)
    result: list[dict] = list(existing)

    # Filter scraped to only allowed intents
    valid_scraped = [
        s for s in scraped
        if s.get("intent") in ALLOWED_INTENTS and s.get("text") and len(s["text"]) >= 8
    ]

    # Deduplicate scraped vs existing
    existing_texts: set[str] = {s["text"].lower().strip() for s in existing}
    new_scraped = [
        s for s in valid_scraped
        if s["text"].lower().strip() not in existing_texts
    ]

    logger.info(f"New (non-duplicate) scraped samples: {len(new_scraped)}")

    # Sort: priority intents first
    def priority_key(s: dict) -> int:
        return 0 if s["intent"] in PRIORITY_INTENTS else 1

    new_scraped.sort(key=priority_key)

    # Add scraped samples respecting caps
    added: Counter = Counter()
    skipped = 0
    for s in new_scraped:
        intent = s["intent"]
        cap = priority_cap if intent in PRIORITY_INTENTS else target_cap
        if counts[intent] < cap:
            result.append(s)
            counts[intent] += 1
            added[intent] += 1
        else:
            skipped += 1

    logger.info(f"Added from scraped: {sum(added.values())} samples")
    logger.info(f"Skipped (cap reached): {skipped}")
    logger.info(f"Per-intent added: {dict(added.most_common())}")

    return result


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--skip-scrape", action="store_true",
                        help="Use existing scraped_*.json files without re-running scrapers")
    parser.add_argument("--dry-run", action="store_true",
                        help="Show counts only, do not write to intent_train.json")
    args = parser.parse_args()

    # Step 1: Run scrapers
    if not args.skip_scrape:
        for script in SCRAPER_SCRIPTS:
            success = run_scraper(script)
            if not success:
                logger.warning(f"Scraper returned non-zero, continuing anyway: {script}")
    else:
        logger.info("--skip-scrape: using existing scraped files")

    # Step 2: Load existing training data
    logger.info(f"Loading existing intent_train.json ...")
    existing = load_json(INTENT_TRAIN_FILE)
    logger.info(f"Existing samples: {len(existing)}")
    existing_counts = Counter(s["intent"] for s in existing)
    logger.info("Existing per-intent:")
    for k, v in sorted(existing_counts.items(), key=lambda x: x[1]):
        logger.info(f"  {k}: {v}")

    # Step 3: Load all scraped files
    all_scraped: list[dict] = []
    for path in SCRAPED_FILES:
        data = load_json(path)
        logger.info(f"Loaded {len(data)} samples from {os.path.basename(path)}")
        all_scraped.extend(data)

    logger.info(f"Total scraped (before dedup): {len(all_scraped)}")

    # Step 4: Merge
    merged = merge_and_cap(existing, all_scraped, TARGET_CAP, PRIORITY_CAP)

    # Step 5: Summary
    merged_counts = Counter(s["intent"] for s in merged)
    logger.info("\n=== FINAL COUNTS ===")
    for k, v in sorted(merged_counts.items(), key=lambda x: x[1]):
        old = existing_counts.get(k, 0)
        gain = v - old
        logger.info(f"  {k}: {old} → {v} (+{gain})")
    logger.info(f"  TOTAL: {len(existing)} → {len(merged)} (+{len(merged)-len(existing)})")

    if args.dry_run:
        logger.info("--dry-run: not writing to intent_train.json")
        return

    # Step 6: Write merged data
    with open(INTENT_TRAIN_FILE, "w", encoding="utf-8") as f:
        json.dump(merged, f, ensure_ascii=False, indent=2)
    logger.info(f"✅ Written to {INTENT_TRAIN_FILE}")

    # Step 7: Cleanup scraped temp files (keep for audit)
    logger.info("Scraped source files kept in datasets/ for audit trail.")


if __name__ == "__main__":
    main()