scripts/data/fetch_new_books.py

#!/usr/bin/env python3
"""
Incremental Book Update Script.

Fetches recently published books from Google Books API and adds them to the local database.
Can be run manually or scheduled via cron for periodic updates.

Usage:
    python scripts/data/fetch_new_books.py [--categories CATEGORIES] [--year YEAR] [--max MAX]

Examples:
    # Fetch new books from current year (default behavior)
    python scripts/data/fetch_new_books.py --categories "fiction" --max 50
    
    # Fetch new books across multiple categories
    python scripts/data/fetch_new_books.py --categories "fiction,mystery,science fiction"
    
    # Explicitly specify year filter
    python scripts/data/fetch_new_books.py --year 2026 --categories "thriller"
    
    # Dry run (show what would be added without actually adding)
    python scripts/data/fetch_new_books.py --dry-run --categories "thriller"
"""
import argparse
import sys
import time
from pathlib import Path
from datetime import datetime
from typing import Optional

# Add project root to path
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
sys.path.insert(0, str(PROJECT_ROOT))

from src.utils import setup_logger
from src.core.rag.web_search import search_new_books_by_category, search_google_books
from src.data.stores.metadata_store import metadata_store
from src.core.recommendation_orchestrator import RecommendationOrchestrator

logger = setup_logger(__name__)

# Default categories to search
DEFAULT_CATEGORIES = [
    "fiction",
    "mystery",
    "thriller",
    "science fiction",
    "fantasy",
    "romance",
    "biography",
    "history",
    "self-help",
    "business",
]


def fetch_trending_books(
    categories: list[str],
    year: Optional[int] = None,
    max_per_category: int = 20,
    dry_run: bool = False,
) -> dict:
    """
    Fetch recently published books from Google Books for given categories.
    
    Args:
        categories: List of book categories to search
        year: Filter by publication year (default: current year)
        max_per_category: Max books to fetch per category
        dry_run: If True, don't actually add books to database
    
    Returns:
        Dict with stats: {added: int, skipped: int, errors: int, books: list}
    """
    if year is None:
        year = datetime.now().year
    
    stats = {
        "added": 0,
        "skipped": 0,
        "errors": 0,
        "books": [],
    }
    
    recommender = None
    if not dry_run:
        recommender = RecommendationOrchestrator()
    
    for category in categories:
        logger.info(f"Fetching books for category: {category} (year >= {year})")
        
        try:
            books = search_new_books_by_category(
                category=category,
                year=year,
                max_results=max_per_category
            )
            
            logger.info(f"  Found {len(books)} books in '{category}'")
            
            for book in books:
                isbn = book.get("isbn13", "")
                if not isbn:
                    continue
                
                # Check if already exists
                if metadata_store.book_exists(isbn):
                    stats["skipped"] += 1
                    continue
                
                if dry_run:
                    logger.info(f"  [DRY RUN] Would add: {book.get('title', 'Unknown')} ({isbn})")
                    stats["books"].append(book)
                    stats["added"] += 1
                else:
                    result = recommender.add_new_book(
                        isbn=isbn,
                        title=book.get("title", ""),
                        author=book.get("authors", "Unknown"),
                        description=book.get("description", ""),
                        category=book.get("simple_categories", category),
                        thumbnail=book.get("thumbnail"),
                        published_date=book.get("publishedDate", ""),
                    )
                    
                    if result:
                        stats["added"] += 1
                        stats["books"].append(book)
                        logger.info(f"  Added: {book.get('title', 'Unknown')} ({isbn})")
                    else:
                        stats["errors"] += 1
                
                # Rate limiting: avoid hitting API limits
                time.sleep(0.1)
            
            # Pause between categories
            time.sleep(0.5)
            
        except Exception as e:
            logger.error(f"Error fetching category '{category}': {e}")
            stats["errors"] += 1
    
    return stats


def fetch_by_query(
    queries: list[str],
    max_per_query: int = 20,
    dry_run: bool = False,
) -> dict:
    """
    Fetch books by specific search queries (e.g., "AI books 2024", "new thriller novels").
    
    Args:
        queries: List of search queries
        max_per_query: Max books per query
        dry_run: If True, don't actually add books
    
    Returns:
        Stats dict
    """
    stats = {
        "added": 0,
        "skipped": 0,
        "errors": 0,
        "books": [],
    }
    
    recommender = None
    if not dry_run:
        recommender = RecommendationOrchestrator()
    
    for query in queries:
        logger.info(f"Searching: {query}")
        
        try:
            books = search_google_books(query, max_results=max_per_query)
            logger.info(f"  Found {len(books)} results")
            
            for book in books:
                isbn = book.get("isbn13", "")
                if not isbn:
                    continue
                
                if metadata_store.book_exists(isbn):
                    stats["skipped"] += 1
                    continue
                
                if dry_run:
                    logger.info(f"  [DRY RUN] Would add: {book.get('title', 'Unknown')}")
                    stats["books"].append(book)
                    stats["added"] += 1
                else:
                    result = recommender.add_new_book(
                        isbn=isbn,
                        title=book.get("title", ""),
                        author=book.get("authors", "Unknown"),
                        description=book.get("description", ""),
                        category=book.get("simple_categories", "General"),
                        thumbnail=book.get("thumbnail"),
                        published_date=book.get("publishedDate", ""),
                    )
                    
                    if result:
                        stats["added"] += 1
                        stats["books"].append(book)
                    else:
                        stats["errors"] += 1
                
                time.sleep(0.1)
            
            time.sleep(0.5)
            
        except Exception as e:
            logger.error(f"Error with query '{query}': {e}")
            stats["errors"] += 1
    
    return stats


def print_stats(stats: dict, dry_run: bool = False):
    """Print summary statistics."""
    prefix = "[DRY RUN] " if dry_run else ""
    print(f"\n{prefix}=== Fetch Complete ===")
    print(f"  Books added:   {stats['added']}")
    print(f"  Books skipped: {stats['skipped']} (already in database)")
    print(f"  Errors:        {stats['errors']}")
    
    if stats["books"] and dry_run:
        print(f"\nBooks that would be added:")
        for book in stats["books"][:10]:
            print(f"  - {book.get('title', 'Unknown')} by {book.get('authors', 'Unknown')}")
        if len(stats["books"]) > 10:
            print(f"  ... and {len(stats['books']) - 10} more")


def main():
    parser = argparse.ArgumentParser(
        description="Fetch new books from Google Books API",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__
    )
    
    parser.add_argument(
        "--categories",
        type=str,
        default=None,
        help="Comma-separated list of categories (default: all common categories)"
    )
    parser.add_argument(
        "--queries",
        type=str,
        default=None,
        help="Comma-separated list of custom search queries"
    )
    parser.add_argument(
        "--year",
        type=int,
        default=None,
        help="Filter by publication year (default: current year)"
    )
    parser.add_argument(
        "--max",
        type=int,
        default=20,
        help="Max books per category/query (default: 20)"
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Show what would be added without actually adding"
    )
    parser.add_argument(
        "--verbose",
        "-v",
        action="store_true",
        help="Enable verbose logging"
    )
    
    args = parser.parse_args()
    
    # Parse categories
    if args.categories:
        categories = [c.strip() for c in args.categories.split(",")]
    else:
        categories = DEFAULT_CATEGORIES
    
    # Parse queries
    queries = None
    if args.queries:
        queries = [q.strip() for q in args.queries.split(",")]
    
    print(f"Book Fetch Configuration:")
    print(f"  Categories: {categories if not queries else 'N/A (using queries)'}")
    print(f"  Queries: {queries or 'N/A (using categories)'}")
    print(f"  Year filter: >= {args.year or datetime.now().year}")
    print(f"  Max per item: {args.max}")
    print(f"  Dry run: {args.dry_run}")
    print()
    
    # Fetch books
    if queries:
        stats = fetch_by_query(
            queries=queries,
            max_per_query=args.max,
            dry_run=args.dry_run,
        )
    else:
        stats = fetch_trending_books(
            categories=categories,
            year=args.year,
            max_per_category=args.max,
            dry_run=args.dry_run,
        )
    
    print_stats(stats, args.dry_run)
    
    return 0 if stats["errors"] == 0 else 1


if __name__ == "__main__":
    sys.exit(main())