backend/tools/dme_extractor.py

import httpx
import re
import json
import asyncio

async def solve_anubis_challenge(url: str, client: httpx.AsyncClient) -> str:
    """Bypass Anubis (Techaro) challenge used by some LATAM repositories"""
    # Simply rotating the User-Agent and setting an Accept-Language often bypasses basic Anubis
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
        'Accept-Language': 'es-PE,es;q=0.9,en-US;q=0.8,en;q=0.7',
        'Sec-Ch-Ua': '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
        'Sec-Ch-Ua-Mobile': '?0',
        'Sec-Ch-Ua-Platform': '"Windows"'
    }
    res = await client.get(url, headers=headers, follow_redirects=True, timeout=15.0)
    return res.text

async def extract_dspace_metadata(url: str) -> dict:
    """Detect DSpace 7 SPA and extract hidden API metadata"""
    async with httpx.AsyncClient(verify=False) as client:
        try:
            html = await solve_anubis_challenge(url, client)
            
            # Detect if it's DSpace 7 Angular
            if 'dspace-angular' in html or 'server/api' in html:
                # Find the handle
                handle_match = re.search(r'handle/(\d+/\d+)', url)
                if handle_match:
                    handle = handle_match.group(1)
                    # Extract the base URL
                    base_url = re.match(r'(https?://[^/]+)', url).group(1)
                    api_url = f"{base_url}/server/api/core/items/search/findByHandle?handle={handle}"
                    
                    api_res = await client.get(api_url, timeout=10.0)
                    if api_res.status_code == 200:
                        data = api_res.json()
                        metadata = data.get('_embedded', {}).get('metadata', {})
                        
                        abstract = ""
                        if 'dc.description.abstract' in metadata:
                            abstract = metadata['dc.description.abstract'][0]['value']
                        
                        # Try to find the bitstream (PDF)
                        pdf_url = ""
                        bundles_link = data.get('_links', {}).get('bundles', {}).get('href')
                        if bundles_link:
                            bundle_res = await client.get(bundles_link)
                            if bundle_res.status_code == 200:
                                bundles = bundle_res.json().get('_embedded', {}).get('bundles', [])
                                for b in bundles:
                                    if b.get('name') == 'ORIGINAL':
                                        bitstreams_link = b.get('_links', {}).get('bitstreams', {}).get('href')
                                        if bitstreams_link:
                                            bits_res = await client.get(bitstreams_link)
                                            if bits_res.status_code == 200:
                                                bits = bits_res.json().get('_embedded', {}).get('bitstreams', [])
                                                for bit in bits:
                                                    if bit.get('bundleName') == 'ORIGINAL' and 'pdf' in bit.get('format', '').lower():
                                                        pdf_url = bit.get('_links', {}).get('content', {}).get('href')
                                                        break
                                        break
                        
                        return {
                            "abstract": abstract,
                            "pdf_url": pdf_url,
                            "enhanced": True
                        }
            
            # Fallback to basic HTML meta tags for older DSpace (xmlui/jspui)
            abstract_match = re.search(r'<meta\s+name="DC\.description\.abstract"\s+content="([^"]+)"', html)
            pdf_match = re.search(r'<meta\s+name="citation_pdf_url"\s+content="([^"]+)"', html)
            
            return {
                "abstract": abstract_match.group(1) if abstract_match else "",
                "pdf_url": pdf_match.group(1) if pdf_match else "",
                "enhanced": bool(abstract_match or pdf_match)
            }
            
        except Exception as e:
            print(f"[DME] Error extracting metadata from {url}: {e}")
            return {"abstract": "", "pdf_url": "", "enhanced": False}

async def enhance_results(results: list) -> list:
    """Run DME on a list of results"""
    tasks = []
    
    async def process_item(r):
        if not r.get("abstract") or len(r["abstract"]) < 50 or "[...]" in r["abstract"] or not r.get("pdfUrl"):
            if r.get("source") in ["ALICIA", "RENATI", "La Referencia", "Bases LATAM"]:
                url = r.get("doi") or "" # Fallback URL is often stored in doi if it's a URI
                if "http" in url:
                    dme_data = await extract_dspace_metadata(url)
                    if dme_data["enhanced"]:
                        if dme_data["abstract"]:
                            r["abstract"] = dme_data["abstract"]
                        if dme_data["pdf_url"]:
                            r["pdfUrl"] = dme_data["pdf_url"]
        return r

    for r in results:
        tasks.append(process_item(r))
        
    return await asyncio.gather(*tasks)