components/api_validator.py

"""
API validation for academic references against CrossRef, arXiv, and OpenAlex.
"""

import requests
import urllib.request
import xml.etree.ElementTree as ET
from urllib.parse import quote
from difflib import SequenceMatcher
import concurrent.futures
import logging
from typing import Dict, List, Optional, Any
from tenacity import (
    retry,
    wait_exponential,
    stop_after_attempt,
    retry_if_exception_type,
    RetryError
)

logger = logging.getLogger(__name__)


class RateLimitError(Exception):
    """Raised when API rate limit is exceeded"""
    pass


class APIValidator:
    """
    Validates academic references against multiple scholarly databases.
    Supports CrossRef, arXiv, and OpenAlex APIs.
    """

    # Configuration
    MAX_WORKERS = 5
    DEFAULT_API_ORDER = ['crossref', 'arxiv', 'openalex']

    # Threshold: Minimum score (0-100) to consider a match valid
    # 70 = balanced threshold allowing minor variations in formatting while avoiding false positives
    SIMILARITY_THRESHOLD = 70

    # Weights: Importance of each field in similarity calculation (must sum to 1.0)
    # Title is most important (80%), author secondary (15%), year and journal minimal (2.5% each)
    SIMILARITY_WEIGHTS = {"title": 0.8, "author": 0.15, "year": 0.025, "journal": 0.025}

    # Retry configuration
    MAX_RETRY_ATTEMPTS = 3
    RETRY_MIN_WAIT = 2  # seconds
    RETRY_MAX_WAIT = 30  # seconds

    @retry(
        wait=wait_exponential(min=RETRY_MIN_WAIT, max=RETRY_MAX_WAIT),
        stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
        retry=retry_if_exception_type((requests.exceptions.RequestException, urllib.error.URLError, RateLimitError)),
        reraise=True
    )
    def _make_request_with_retry(self, url: str, use_urllib: bool = False, **kwargs):
        """
        Makes HTTP request with automatic retry on failure.

        Args:
            url: The URL to request
            use_urllib: If True, uses urllib instead of requests
            **kwargs: Additional arguments passed to request method

        Returns:
            Response object or data

        Raises:
            RateLimitError: If rate limited after all retries
            requests.exceptions.RequestException: On other request failures
        """
        if use_urllib:
            logger.debug(f"Making urllib request to: {url[:100]}...")
            response = urllib.request.urlopen(url, **kwargs)
            return response.read()
        else:
            logger.debug(f"Making requests call to: {url[:100]}...")
            response = requests.get(url, **kwargs)

            # Check for rate limiting
            if response.status_code == 429:
                retry_after = response.headers.get('Retry-After', 'unknown')
                logger.warning(f"Rate limited (429). Retry-After: {retry_after}")
                raise RateLimitError(f"API rate limit exceeded. Retry after: {retry_after}s")

            response.raise_for_status()
            return response

    def validate_concurrent(self, references: Optional[List[Dict[str, Any]]] = None) -> List[Dict[str, Any]]:
        """
        Processes multiple references concurrently.

        Args:
            references: List of references to check.

        Returns:
            List of validation results.
        """
        if references is None:
            references = []

        if not references:
            logger.warning("No references provided for validation")
            return []

        logger.info(f"Starting concurrent validation of {len(references)} references")

        with concurrent.futures.ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
            future_to_reference = {
                executor.submit(self.validate_single, ref): i
                for i, ref in enumerate(references)
            }

            results = [None] * len(references)
            for future in concurrent.futures.as_completed(future_to_reference):
                idx = future_to_reference[future]
                try:
                    results[idx] = future.result()
                    logger.debug(f"Reference {idx} validated successfully")
                except Exception as exc:
                    logger.error(f"Reference {idx} validation failed: {exc}")
                    results[idx] = {
                        "exists": False,
                        "error": str(exc),
                        "message": "Error occurred during validation",
                        "query": references[idx]
                    }

        logger.info(f"Completed validation of {len(references)} references")
        return results

    def validate_single(self, reference: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
        """
        Validates a single reference through multiple APIs sequentially.

        Args:
            reference: Dictionary containing reference details.

        Returns:
            Dictionary with validation results.
        """
        if reference is None:
            reference = {}

        title = reference.get('title')
        first_author = reference.get('first_author')
        year = reference.get('year')
        journal = reference.get('journal')
        reference_text = reference.get('reference')

        if not title:
            logger.warning("Reference validation attempted without title")
            return {
                "exists": False,
                "message": "No title provided",
                "query": reference
            }

        # Get API order or use default
        api_order = reference.get('check_api_order', self.DEFAULT_API_ORDER)
        logger.debug(f"Validating reference: {title[:50]}... using APIs: {api_order}")

        result = {
            "exists": False,
            "source": None,
            "message": "No matching reference found",
            "query": reference
        }

        # Try each API in order until we find a match
        for api in api_order:
            api_result = None

            if api == 'crossref':
                api_result = self._validate_crossref(title, first_author, year, journal, reference_text)
            elif api == 'arxiv':
                api_result = self._validate_arxiv(title, first_author, year, journal, reference_text)
            elif api == 'openalex':
                api_result = self._validate_openalex(title, first_author, year, journal, reference_text)

            if api_result and api_result.get("exists"):
                result["exists"] = True
                result["source"] = api
                result["message"] = api_result.get("message", "Found matching reference")
                result["best_match"] = api_result.get("best_match")
                result["matches"] = api_result.get("matches")
                break

        return result

    def _validate_crossref(self, title: str, first_author: Optional[str] = None,
                          year: Optional[str] = None, journal: Optional[str] = None,
                          reference_text: Optional[str] = None) -> Dict[str, Any]:
        """Validates against CrossRef API."""
        try:
            query = {"title": title, "first_author": first_author, "year": year, "journal": journal}

            # Sanitize reference_text to prevent injection attacks
            search_text = reference_text if reference_text else title
            if not search_text or not search_text.strip():
                logger.warning("Empty search text for CrossRef validation")
                return {
                    "exists": False,
                    "message": "No search text provided",
                    "query": query
                }

            # URL encode the search text
            encoded_search = quote(search_text.strip(), safe='')
            url = f"https://api.crossref.org/works?query.bibliographic={encoded_search}&rows=3"

            logger.debug(f"CrossRef API call for: {title[:50]}...")

            try:
                api_query = self._make_request_with_retry(url, timeout=10)
                data = api_query.json()
            except RetryError as e:
                logger.error(f"CrossRef API failed after {self.MAX_RETRY_ATTEMPTS} retries: {e}")
                return {
                    "exists": False,
                    "error": "max_retries_exceeded",
                    "message": f"CrossRef API unavailable after {self.MAX_RETRY_ATTEMPTS} retry attempts",
                    "query": query
                }
            except RateLimitError as e:
                logger.error(f"CrossRef rate limit exceeded: {e}")
                return {
                    "exists": False,
                    "error": "rate_limit",
                    "message": "CrossRef API rate limit exceeded. Please try again later.",
                    "query": query
                }

            results = []
            if api_query.status_code == 200:
                results.extend(data.get('message', {}).get('items', []))

            extracted_results = []
            for item in results:
                # Extract relevant data from CrossRef response
                authors = []
                for author in item.get('author', []):
                    if 'given' in author and 'family' in author:
                        authors.append(f"{author['given']} {author['family']}")
                    elif 'family' in author:
                        authors.append(author['family'])

                extracted_data = {
                    'title': item.get('title', [''])[0] if item.get('title') else '',
                    'authors': authors,
                    'year': item.get('published-print', {}).get('date-parts', [[None]])[0][0] or
                            item.get('published-online', {}).get('date-parts', [[None]])[0][0],
                    'journal': item.get('container-title', [''])[0] if item.get('container-title') else '',
                    'link': item.get('URL'),
                    'url': item.get('URL')
                }

                extracted_results.append(extracted_data)

            return self._calculate_matches(extracted_results, query, self.SIMILARITY_WEIGHTS, self.SIMILARITY_THRESHOLD)

        except requests.exceptions.Timeout:
            logger.error(f"CrossRef API timeout for: {title[:50]}")
            return {
                "exists": False,
                "error": "API timeout",
                "message": "CrossRef API request timed out",
                "query": query
            }
        except requests.exceptions.RequestException as e:
            logger.error(f"CrossRef API request failed: {e}")
            return {
                "exists": False,
                "error": str(e),
                "message": "CrossRef API request failed",
                "query": query
            }
        except Exception as e:
            logger.exception(f"Unexpected error in CrossRef validation: {e}")
            return {
                "exists": False,
                "error": str(e),
                "message": "Error occurred during validation",
                "query": query
            }

    def _validate_arxiv(self, title: str, first_author: Optional[str] = None,
                       year: Optional[str] = None, journal: Optional[str] = None,
                       reference_text: Optional[str] = None) -> Dict[str, Any]:
        """Validates against arXiv API."""
        try:
            query = {"title": title, "first_author": first_author, "year": year, "journal": journal}

            if not title or not title.strip():
                logger.warning("Empty title for arXiv validation")
                return {
                    "exists": False,
                    "message": "No title provided",
                    "query": query
                }

            # Sanitize and encode title
            encoded_title = quote(title.strip())
            url = f'http://export.arxiv.org/api/query?search_query=title:{encoded_title}&start=0&max_results=5'

            logger.debug(f"arXiv API call for: {title[:50]}...")

            try:
                data = self._make_request_with_retry(url, use_urllib=True, timeout=10)
            except RetryError as e:
                logger.error(f"arXiv API failed after {self.MAX_RETRY_ATTEMPTS} retries: {e}")
                return {
                    "exists": False,
                    "error": "max_retries_exceeded",
                    "message": f"arXiv API unavailable after {self.MAX_RETRY_ATTEMPTS} retry attempts",
                    "query": query
                }
            except RateLimitError as e:
                logger.error(f"arXiv rate limit exceeded: {e}")
                return {
                    "exists": False,
                    "error": "rate_limit",
                    "message": "arXiv API rate limit exceeded. Please try again later.",
                    "query": query
                }

            namespaces = {
                'atom': 'http://www.w3.org/2005/Atom',
                'arxiv': 'http://arxiv.org/schemas/atom'
            }

            root = ET.fromstring(data)
            entries = root.findall('atom:entry', namespaces)

            extracted_results = []
            for entry in entries:
                title_elem = entry.find('atom:title', namespaces)
                authors = entry.findall('atom:author/atom:name', namespaces)
                published = entry.find('atom:published', namespaces)
                link = entry.find('atom:id', namespaces)

                extracted_data = {
                    'title': title_elem.text.strip() if title_elem is not None else '',
                    'authors': [author.text for author in authors],
                    'year': published.text[:4] if published is not None else None,
                    'journal': 'arXiv',
                    'link': link.text if link is not None else None,
                    'url': link.text if link is not None else None
                }

                extracted_results.append(extracted_data)

            return self._calculate_matches(extracted_results, query, self.SIMILARITY_WEIGHTS, self.SIMILARITY_THRESHOLD)

        except urllib.error.URLError as e:
            logger.error(f"arXiv API connection error: {e}")
            return {
                "exists": False,
                "error": str(e),
                "message": "arXiv API connection failed",
                "query": query
            }
        except Exception as e:
            logger.exception(f"Unexpected error in arXiv validation: {e}")
            return {
                "exists": False,
                "error": str(e),
                "message": "Error occurred during validation",
                "query": query
            }

    def _validate_openalex(self, title: str, first_author: Optional[str] = None,
                          year: Optional[str] = None, journal: Optional[str] = None,
                          reference_text: Optional[str] = None) -> Dict[str, Any]:
        """Validates against OpenAlex API."""
        try:
            query = {"title": title, "first_author": first_author, "year": year, "journal": journal}

            if not title or not title.strip():
                logger.warning("Empty title for OpenAlex validation")
                return {
                    "exists": False,
                    "message": "No title provided",
                    "query": query
                }

            # Sanitize and encode title
            encoded_title = quote(title.strip())
            url = f"https://api.openalex.org/works?search=\"{encoded_title}\""

            logger.debug(f"OpenAlex API call for: {title[:50]}...")

            try:
                response = self._make_request_with_retry(url, timeout=10)
                data = response.json()
            except RetryError as e:
                logger.error(f"OpenAlex API failed after {self.MAX_RETRY_ATTEMPTS} retries: {e}")
                return {
                    "exists": False,
                    "error": "max_retries_exceeded",
                    "message": f"OpenAlex API unavailable after {self.MAX_RETRY_ATTEMPTS} retry attempts",
                    "query": query
                }
            except RateLimitError as e:
                logger.error(f"OpenAlex rate limit exceeded: {e}")
                return {
                    "exists": False,
                    "error": "rate_limit",
                    "message": "OpenAlex API rate limit exceeded. Please try again later.",
                    "query": query
                }

            if not data or "results" not in data:
                return {
                    "exists": False,
                    "message": "No results from OpenAlex",
                    "query": query
                }

            extracted_results = []
            for item in data.get('results', []):
                authors = []
                for authorship in item.get('authorships', []):
                    author_info = authorship.get('author', {})
                    display_name = author_info.get('display_name')
                    if display_name:
                        authors.append(display_name)

                # Safely extract journal name - primary_location or source might be None
                journal = ''
                primary_location = item.get('primary_location')
                if primary_location:
                    source = primary_location.get('source')
                    if source:
                        journal = source.get('display_name', '')

                extracted_data = {
                    'title': item.get('title', ''),
                    'authors': authors,
                    'year': item.get('publication_year'),
                    'journal': journal,
                    'link': item.get('doi'),
                    'url': item.get('doi')
                }

                extracted_results.append(extracted_data)

            return self._calculate_matches(extracted_results, query, self.SIMILARITY_WEIGHTS, self.SIMILARITY_THRESHOLD)

        except requests.exceptions.Timeout:
            logger.error(f"OpenAlex API timeout for: {title[:50]}")
            return {
                "exists": False,
                "error": "API timeout",
                "message": "OpenAlex API request timed out",
                "query": query
            }
        except requests.exceptions.RequestException as e:
            logger.error(f"OpenAlex API request failed: {e}")
            return {
                "exists": False,
                "error": str(e),
                "message": "OpenAlex API request failed",
                "query": query
            }
        except Exception as e:
            logger.exception(f"Unexpected error in OpenAlex validation: {e}")
            return {
                "exists": False,
                "error": str(e),
                "message": "Error occurred during validation",
                "query": query
            }

    def _calculate_matches(self, extracted_results: List[Dict], query: Dict,
                          weights: Optional[Dict] = None, threshold: int = 70) -> Dict[str, Any]:
        """
        Calculates similarity matches between query and extracted results.

        Args:
            extracted_results: List of results from API
            query: Original query parameters
            weights: Similarity weights for different fields
            threshold: Minimum score threshold

        Returns:
            Dictionary with match information
        """
        if weights is None:
            weights = {"title": 0.8, "author": 0.1, "year": 0.05, "journal": 0.05}

        matches = []

        for data in extracted_results:
            title_similarity = 0
            author_similarity = 0
            year_similarity = 0
            journal_similarity = 0

            # Title similarity
            if query.get("title") and data.get("title"):
                title_similarity = SequenceMatcher(
                    None, query["title"].lower(), data["title"].lower()
                ).ratio() * 100

            # Author similarity
            if query.get("first_author") and data.get("authors"):
                first_author_query = query["first_author"].lower()
                author_similarities = [
                    SequenceMatcher(None, first_author_query, author.lower()).ratio() * 100
                    for author in data["authors"]
                ]
                author_similarity = max(author_similarities) if author_similarities else 0

            # Year similarity
            year = query.get("year")
            if year and data.get("year") and year == str(data["year"]):
                year_similarity = 100

            # Journal similarity
            if query.get("journal") and data.get("journal"):
                journal_similarity = SequenceMatcher(
                    None, query["journal"].lower(), data["journal"].lower()
                ).ratio() * 100

            # Calculate weighted overall score
            overall_score = (
                title_similarity * weights.get("title", 0) +
                author_similarity * weights.get("author", 0) +
                year_similarity * weights.get("year", 0) +
                journal_similarity * weights.get("journal", 0)
            )

            match_info = {
                **data,
                "overall_score": overall_score,
                "title_similarity": title_similarity,
                "author_similarity": author_similarity,
                "year_similarity": year_similarity,
                "journal_similarity": journal_similarity,
                "is_match": overall_score >= threshold
            }

            matches.append(match_info)

        # Sort by overall score
        matches.sort(key=lambda x: x["overall_score"], reverse=True)

        best_match = matches[0] if matches else None
        exists = best_match and best_match["is_match"]

        return {
            "exists": exists,
            "message": "Found matching reference" if exists else "No sufficiently similar references found",
            "matches": matches,
            "best_match": best_match
        }