backend/app/test_historical_scraper.py

"""
Tests for Historical Scraper Tool — DataBus integration
=====================================================
"""

import sys
sys.path.insert(0, '/root/backend')
sys.path.insert(0, '/root/backend/app')

import asyncio
import json
from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock, patch

import pytest


class TestHistoricalScraperTool:
    """Test suite for historical_scraper_tool.py"""

    def test_register_historical_chains_returns_dict(self):
        """Test that register_historical_chains returns a dictionary of chains."""
        from app.historical_scraper_tool import register_historical_chains
        
        chains = register_historical_chains()
        
        assert isinstance(chains, dict)
        assert "historical_ingest" in chains
        assert "historical_sources" in chains
        assert "defi_hacks" in chains
        assert "scam_reports" in chains

    def test_register_historical_chains_provider_count(self):
        """Test that each chain has at least one provider."""
        from app.historical_scraper_tool import register_historical_chains
        
        chains = register_historical_chains()
        
        for name, chain in chains.items():
            assert chain.data_type == name, f"Chain {name} has wrong data_type"
            assert len(chain.providers) >= 1, f"Chain {name} has no providers"

    def test_sources_dict_structure(self):
        """Test that SOURCES dict in rag_historical has expected structure."""
        from app.rag_historical import SOURCES
        
        expected_sources = ["rekt_db", "chainabuse", "slowmist_hacked", "rekt_news", "trm_crime_report"]
        
        for source in expected_sources:
            assert source in SOURCES, f"Missing source: {source}"
            assert "name" in SOURCES[source]
            assert "collection" in SOURCES[source]
            assert "url" in SOURCES[source]

    @pytest.mark.asyncio
    async def test_fetch_historical_sources(self):
        """Test fetching historical sources list."""
        from app.historical_scraper_tool import _fetch_historical_sources
        
        result = await _fetch_historical_sources()
        assert result is not None
        assert "sources" in result
        assert "total_sources" in result
        assert result["total_sources"] >= 5

    @pytest.mark.asyncio
    async def test_fetch_historical_ingest_unknown_source(self):
        """Test that unknown source returns error."""
        from app.historical_scraper_tool import _fetch_historical_ingest
        
        result = await _fetch_historical_ingest(source_id="unknown_source")
        
        assert "error" in result


class TestRagHistoricalScrapers:
    """Test suite for rag_historical.py scrapers"""

    @pytest.mark.asyncio
    async def test_scrape_rekt_db_returns_list(self):
        """Test that scrape_rekt_db returns a list of documents."""
        from app.rag_historical import scrape_rekt_db
        
        # This may fail if API is unreachable, but should return empty list not crash
        result = await scrape_rekt_db()
        
        assert isinstance(result, list)

    @pytest.mark.asyncio
    async def test_scrape_chainabuse_returns_list(self):
        """Test that scrape_chainabuse returns a list of documents."""
        from app.rag_historical import scrape_chainabuse
        
        result = await scrape_chainabuse()
        
        assert isinstance(result, list)

    @pytest.mark.asyncio
    async def test_ingest_historical_source_unknown(self):
        """Test that unknown source returns error dict."""
        from app.rag_historical import ingest_historical_source
        
        result = await ingest_historical_source("nonexistent_source")
        
        assert "error" in result
        assert "Unknown source" in result["error"]

    @pytest.mark.asyncio
    async def test_ingest_all_historical_structure(self):
        """Test structure of ingest_all_historical result."""
        from app.rag_historical import ingest_all_historical
        
        result = await ingest_all_historical()
        
        assert "status" in result
        assert "sources" in result
        assert "timestamp" in result


class TestIntegration:
    """Integration tests for DataBus chains"""

    def test_chains_registered_in_providers(self):
        """Test that historical chains are registered in build_provider_chains."""
        from app.databus.providers import build_provider_chains
        
        chains = build_provider_chains()
        
        # Check that our historical chains were added
        assert "historical_ingest" in chains
        assert "historical_sources" in chains
        assert "defi_hacks" in chains
        assert "scam_reports" in chains

    def test_router_list_mode_ok_updated(self):
        """Test that router LIST_MODE_OK includes historical chains."""
        from databus.router import LIST_MODE_OK
        
        assert "historical_ingest" in LIST_MODE_OK
        assert "historical_sources" in LIST_MODE_OK
        assert "defi_hacks" in LIST_MODE_OK
        assert "scam_reports" in LIST_MODE_OK


if __name__ == "__main__":
    pytest.main([__file__, "-v"])