backend/app/historical_scraper_tool.py

"""
RAG Historical Scraper Tool — Wire historical sources into DataBus
================================================================

Registers Rekt DB, Chainabuse, SlowMist, Immunefi scrapers as DataBus chains
for on-demand and scheduled ingestion into RAG collections.

This fills the major gap identified in RAG_MODERNIZATION.md:
"Firehose sources not wired (cadences defined, fetchers missing)"

DataBus endpoints added:
- POST /api/v1/databus/fetch/historical_ingest
- POST /api/v1/databus/fetch/historical_ingest/{source_id}
- GET /api/v1/databus/fetch/historical_sources

Sources: rekt_db (DeFi hacks), chainabuse (scam reports), slowmist, immunefi, trm_crime_report
"""

import asyncio
import json
import logging
from datetime import UTC, datetime
from typing import Any

logger = logging.getLogger("historical_scraper_tool")

# Import the scrapers from rag_historical
try:
    from app.rag_historical import ingest_historical_source, ingest_all_historical, SOURCES
except ImportError:
    # Will be available when run as module
    ingest_historical_source = None
    ingest_all_historical = None
    SOURCES = {}


# ── Provider Functions ───────────────────────────────────────────────


async def _fetch_historical_ingest(source_id: str = "all", **kwargs) -> dict | None:
    """Run historical ingestion for a specific source or all sources.
    
    source_id: "rekt_db", "chainabuse", "slowmist_hacked", "trm_crime_report", or "all"
    """
    # Import here to allow circular imports
    from app.rag_historical import ingest_historical_source, ingest_all_historical, SOURCES
    
    force = kwargs.get("force", False)
    
    if source_id == "all":
        result = await ingest_all_historical()
    elif source_id in SOURCES:
        result = await ingest_historical_source(source_id)
    else:
        return {"error": f"Unknown source: {source_id}", "valid_sources": list(SOURCES.keys())}
    
    return result


async def _fetch_historical_sources(**kwargs) -> dict | None:
    """List all available historical sources and their configuration."""
    from app.rag_historical import SOURCES
    
    sources_info = []
    for source_id, info in SOURCES.items():
        sources_info.append({
            "id": source_id,
            "name": info.get("name", source_id),
            "url": info.get("url", ""),
            "collection": info.get("collection", "unknown"),
            "cadence": info.get("cadence", "unknown"),
            "content_type": info.get("content_type", "unknown"),
            "description": info.get("description", ""),
        })
    
    return {
        "sources": sources_info,
        "total_sources": len(sources_info),
        "timestamp": datetime.now(UTC).isoformat(),
    }


async def _fetch_defi_hacks(limit: int = 50, **kwargs) -> dict | None:
    """Fetch recent DeFi hacks from Rekt DB (cached 24h)."""
    from app.rag_historical import scrape_rekt_db
    
    try:
        docs = await scrape_rekt_db()
        hacks = docs[:limit] if docs else []
        
        return {
            "hacks": hacks,
            "count": len(hacks),
            "source": "rekt_db",
            "cached": True,  # Will be cached by DataBus
        }
    except Exception as e:
        logger.warning(f"DeFi hacks fetch failed: {e}")
        return None


async def _fetch_scam_reports(limit: int = 50, **kwargs) -> dict | None:
    """Fetch recent scam reports from Chainabuse (cached 24h)."""
    from app.rag_historical import scrape_chainabuse
    
    try:
        docs = await scrape_chainabuse()
        reports = docs[:limit] if docs else []
        
        return {
            "reports": reports,
            "count": len(reports),
            "source": "chainabuse",
        }
    except Exception as e:
        logger.warning(f"Scam reports fetch failed: {e}")
        return None


# ── Chain Registration ───────────────────────────────────────────────


def register_historical_chains() -> dict[str, Any]:
    """Register historical scraper chains with DataBus.
    
    Must be called during app initialization.
    Returns dict of chain_name -> ProviderChain.
    """
    from app.databus.providers import Provider, ProviderChain, ProviderTier
    
    chains = {}
    
    # Historical ingestion (admin only, background task)
    chains["historical_ingest"] = ProviderChain(
        data_type="historical_ingest",
        description="Ingest historical scam/hack data into RAG (Rekt DB, Chainabuse, etc.)",
        providers=[
            Provider(
                "historical_scraper",
                ProviderTier.LOCAL,
                _fetch_historical_ingest,
                weight=10.0,
                rate_limit_rps=0.1,  # Very slow - don't spam API
                is_local=True,
            ),
        ],
    )
    
    # List available sources
    chains["historical_sources"] = ProviderChain(
        data_type="historical_sources",
        description="List all historical data sources",
        providers=[
            Provider(
                "sources_lister",
                ProviderTier.LOCAL,
                _fetch_historical_sources,
                weight=10.0,
                rate_limit_rps=5.0,
                is_local=True,
            ),
        ],
    )
    
    # DeFi hacks endpoint (cached)
    chains["defi_hacks"] = ProviderChain(
        data_type="defi_hacks",
        description="Recent DeFi hacks from Rekt DB",
        providers=[
            Provider(
                "rekt_db_scraper",
                ProviderTier.LOCAL,
                _fetch_defi_hacks,
                weight=10.0,
                rate_limit_rps=0.1,
                is_local=True,
            ),
        ],
    )
    
    # Scam reports endpoint (cached)
    chains["scam_reports"] = ProviderChain(
        data_type="scam_reports",
        description="Scam reports from Chainabuse",
        providers=[
            Provider(
                "chainabuse_scraper",
                ProviderTier.LOCAL,
                _fetch_scam_reports,
                weight=10.0,
                rate_limit_rps=0.1,
                is_local=True,
            ),
        ],
    )
    
    logger.info(
        f"Historical scraper chains registered: "
        f"{list(chains.keys())}"
    )
    
    return chains


# ── Quick CLI Test ───────────────────────────────────────────────────


if __name__ == "__main__":
    import sys
    
    async def main():
        if len(sys.argv) > 1:
            source = sys.argv[1]
            result = await _fetch_historical_ingest(source)
            print(json.dumps(result, indent=2, default=str))
        else:
            sources = await _fetch_historical_sources()
            print(json.dumps(sources, indent=2, default=str))
    
    asyncio.run(main())