app/main.py

# FastAPI Application Entry Point
from app.config import settings

from fastapi import FastAPI, HTTPException, Query, Depends, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse, FileResponse
from fastapi.staticfiles import StaticFiles
from typing import Optional
import os
from slowapi import Limiter, _rate_limit_exceeded_handler
from slowapi.util import get_remote_address
from slowapi.errors import RateLimitExceeded
from app.models import SourceConfig, SourceType
from app.grabber import GitHubGrabber
from app.sources import SourceRegistry
from app.database import init_db, AsyncSessionLocal, get_db, AsyncSession
from app.db_storage import db_storage
from app.routers import auth, sources, proxies, notifications, validation, admin
from app.dependencies import require_admin
from app.db_models import User
from app.background_validator import background_validation_worker
import asyncio
import logging

# Configure logging
logging.basicConfig(
    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# Configure rate limiting
limiter = Limiter(key_func=get_remote_address)

app = FastAPI(
    title=settings.PROJECT_NAME, openapi_url=f"{settings.API_V1_STR}/openapi.json"
)

# Add rate limiter state
app.state.limiter = limiter
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)


# Global exception handler to prevent leaking internal errors
@app.exception_handler(Exception)
async def global_exception_handler(request: Request, exc: Exception):
    """

    Catch all unhandled exceptions and return a safe error message.

    Log the full error details for debugging.

    """
    import uuid
    import traceback

    error_id = str(uuid.uuid4())[:8]
    logger.error(
        f"Unhandled exception [{error_id}]: {exc}",
        exc_info=True,
        extra={
            "error_id": error_id,
            "path": request.url.path,
            "method": request.method,
            "traceback": traceback.format_exc(),
        },
    )

    return JSONResponse(
        status_code=500,
        content={
            "detail": "An internal error occurred. Please try again later.",
            "error_id": error_id,
        },
    )


# CORS middleware configuration - support HF Spaces, GitHub Pages, and local development
app.add_middleware(
    CORSMiddleware,
    allow_origins=[
        settings.FRONTEND_URL,
        settings.API_URL,
        "http://localhost:3000",
        "http://localhost:8000",
        "https://*.hf.space",
        "https://*.spaces.huggingface.tech",
        "https://*.github.io",  # GitHub Pages support
    ],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Serve Next.js frontend static files (built with standalone output)
# Check multiple possible locations for the frontend build
frontend_paths = [
    "/app/frontend",
    "/app/1proxy-frontend",
    os.path.join(os.path.dirname(__file__), "../../1proxy-frontend"),
]

frontend_path = None
for fp in frontend_paths:
    if os.path.exists(os.path.join(fp, "server.js")):
        frontend_path = fp
        break

if frontend_path:
    logger.info(f"📦 Serving frontend from: {frontend_path}")
    # Mount the Next.js build output
    app.mount(
        "/static",
        StaticFiles(directory=os.path.join(frontend_path, ".next/static")),
        name="static",
    )
    app.mount("/_next", StaticFiles(directory=frontend_path), name="next")

    @app.get("/favicon.ico")
    async def favicon():
        favicon_path = os.path.join(frontend_path, "public/favicon.ico")
        if os.path.exists(favicon_path):
            return FileResponse(favicon_path)
        return JSONResponse(status_code=204, content={})


app.include_router(auth.router)
app.include_router(sources.router)
app.include_router(proxies.router)
app.include_router(notifications.router)
app.include_router(validation.router)
app.include_router(admin.router)

from app.admin.scraping_admin import router as scraping_admin_router

app.include_router(scraping_admin_router)

grabber = GitHubGrabber()


@app.on_event("startup")
async def startup():
    await init_db()

    async with AsyncSessionLocal() as session:
        try:
            admin_user = await db_storage.get_or_create_user(
                session=session,
                oauth_provider="local",
                oauth_id="admin",
                email="admin@1proxy.local",
                username="admin",
                role="admin",
            )
            await db_storage.seed_admin_sources(session, admin_user.id)
            await session.commit()
            logger.info(
                f"✅ Admin user created/verified: {admin_user.username} (ID: {admin_user.id})"
            )
            logger.info("✅ Admin sources seeded")
        except Exception as e:
            logger.warning(f"⚠️  Startup error (non-critical): {e}")
            await session.rollback()

    # STARTUP STABILIZER: Wait for HF Space to pass health check before spawning workers
    async def delayed_workers():
        logger.info("⏳ Stabilizer: Waiting 15s before starting background workers...")
        await asyncio.sleep(15)

        logger.info("🚀 Stabilizer: Spawning background workers...")
        # Start validation worker with reduced batch size for HF
        asyncio.create_task(
            background_validation_worker(batch_size=20, interval_seconds=60)
        )

        # Import and start auto-scraper
        from app.background_validator import background_scraper_worker

        asyncio.create_task(background_scraper_worker(interval_minutes=10))
        logger.info("✅ Stabilizer: Background workers active")

    asyncio.create_task(delayed_workers())


@app.get("/")
async def root():
    return {
        "name": "1proxy API",
        "version": "2.0.0",
        "status": "running",
        "features": {
            "multi_user": True,
            "oauth": ["github", "google"],
            "advanced_filtering": True,
            "export_formats": ["txt", "json", "csv"],
        },
        "endpoints": {
            "health": "/health",
            "auth": "/auth/*",
            "my_sources": "/api/v1/my-sources",
            "advanced_search": "/api/v1/proxies/advanced",
            "export": "/api/v1/proxies/export",
            "public_sources": "/api/v1/sources",
        },
    }


@app.get("/health")
async def health_check(session: AsyncSession = Depends(get_db)):
    proxy_count = await db_storage.count_proxies(session)
    source_count = await db_storage.count_sources(session)
    user_count = await db_storage.count_users(session)
    return {
        "status": "healthy",
        "database": "connected",
        "proxies": proxy_count,
        "sources": source_count,
        "users": user_count,
    }


@app.post("/api/v1/proxies/scrape", response_model=dict)
async def scrape_proxies(

    source: SourceConfig, current_user: User = Depends(require_admin)

):
    async with AsyncSessionLocal() as session:
        try:
            proxies = await grabber.extract_proxies(source)
            proxies_data = []
            for p in proxies:
                data = p.model_dump() if hasattr(p, "model_dump") else p.__dict__
                proxies_data.append(
                    {
                        "url": f"{data.get('protocol', 'http')}://{data.get('ip')}:{data.get('port')}",
                        "protocol": data.get("protocol", "http"),
                        "ip": data.get("ip"),
                        "port": data.get("port"),
                        "country_code": data.get("country_code"),
                        "country_name": data.get("country_name"),
                        "city": data.get("city"),
                        "latency_ms": data.get("latency_ms"),
                        "speed_mbps": data.get("speed_mbps"),
                        "anonymity": data.get("anonymity"),
                        "proxy_type": data.get("proxy_type"),
                    }
                )
            added = await db_storage.add_proxies(session, proxies_data)

            validation_results = await db_storage.validate_and_update_proxies(
                session, limit=min(added, 100)
            )

            return {
                "source": str(source.url),
                "scraped": len(proxies),
                "added": added,
                "validated": validation_results["validated"],
                "failed": validation_results["failed"],
                "total": await db_storage.count_proxies(session),
            }
        except FileNotFoundError as e:
            raise HTTPException(status_code=404, detail=str(e))
        except asyncio.TimeoutError:
            raise HTTPException(status_code=504, detail="Request timeout")
        except Exception as e:
            raise HTTPException(status_code=500, detail=str(e))


@app.get("/api/v1/proxies", response_model=dict)
async def list_proxies(

    protocol: Optional[str] = Query(

        None, description="Filter by protocol (http, vmess, vless, trojan, shadowsocks)"

    ),

    limit: int = Query(10, ge=1, le=100, description="Number of results"),

    offset: int = Query(0, ge=0, description="Offset for pagination"),

    session: AsyncSession = Depends(get_db),

):
    proxies, total = await db_storage.get_proxies(
        session=session, protocol=protocol, limit=limit, offset=offset, is_working=True
    )

    proxy_list = []
    for p in proxies:
        proxy_list.append(
            {
                "id": p.id,
                "url": p.url,
                "protocol": p.protocol,
                "ip": p.ip,
                "port": p.port,
                "country_code": p.country_code,
                "country_name": p.country_name,
                "city": p.city,
                "latency_ms": p.latency_ms,
                "speed_mbps": p.speed_mbps,
                "anonymity": p.anonymity,
                "quality_score": p.quality_score,
                "is_working": p.is_working,
                "last_validated": p.last_validated.isoformat()
                if p.last_validated
                else None,
                "source": str(p.source_id),
            }
        )

    return {
        "total": total,
        "count": len(proxies),
        "offset": offset,
        "limit": limit,
        "proxies": proxy_list,
    }

    return {
        "total": total,
        "count": len(proxies),
        "offset": offset,
        "limit": limit,
        "proxies": proxy_list,
    }


@app.get("/api/v1/stats")
async def get_stats(session: AsyncSession = Depends(get_db)):
    stats = await db_storage.get_stats(session)
    user_count = await db_storage.count_users(session)
    source_count = await db_storage.count_sources(session)

    stats["total_users"] = user_count
    stats["total_sources"] = source_count

    return stats


@app.post("/api/v1/proxies/demo")
async def demo_scrape(current_user: User = Depends(require_admin)):
    async with AsyncSessionLocal() as session:
        source = SourceConfig(
            url="https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt",
            type=SourceType.GITHUB_RAW,
        )
        try:
            proxies = await grabber.extract_proxies(source)
            proxies_data = []
            sample_list = []
            for p in proxies:
                data = p.model_dump() if hasattr(p, "model_dump") else p.__dict__
                proxy_data = {
                    "url": f"{data.get('protocol', 'http')}://{data.get('ip')}:{data.get('port')}",
                    "protocol": data.get("protocol", "http"),
                    "ip": data.get("ip"),
                    "port": data.get("port"),
                    "country_code": data.get("country_code"),
                    "country_name": data.get("country_name"),
                    "city": data.get("city"),
                    "latency_ms": data.get("latency_ms"),
                    "speed_mbps": data.get("speed_mbps"),
                    "anonymity": data.get("anonymity"),
                    "proxy_type": data.get("proxy_type"),
                }
                proxies_data.append(proxy_data)
                if len(sample_list) < 5:
                    sample_list.append(data)

            added = await db_storage.add_proxies(session, proxies_data)

            validation_results = await db_storage.validate_and_update_proxies(
                session, limit=min(added, 50)
            )

            return {
                "message": "Demo scrape completed",
                "source": str(source.url),
                "scraped": len(proxies),
                "added": added,
                "validated": validation_results["validated"],
                "failed": validation_results["failed"],
                "total_stored": await db_storage.count_proxies(session),
                "sample": sample_list,
            }
        except Exception as e:
            raise HTTPException(status_code=500, detail=str(e))


@app.get("/api/v1/sources")
async def list_sources(session: AsyncSession = Depends(get_db)):
    sources = await db_storage.get_sources(session, enabled_only=False)
    return {
        "total": len(sources),
        "enabled": len([s for s in sources if s.enabled]),
        "sources": [
            {
                "id": s.id,
                "url": s.url,
                "type": s.type,
                "enabled": s.enabled,
                "name": s.name,
                "is_admin_source": s.is_admin_source,
                "validated": s.validated,
                "total_scraped": s.total_scraped,
            }
            for s in sources
        ],
    }


@app.post("/api/v1/proxies/scrape-all")
async def scrape_all_sources(current_user: User = Depends(require_admin)):
    async with AsyncSessionLocal() as session:
        sources = SourceRegistry.get_enabled_sources()
        results = []
        total_scraped = 0
        total_added = 0
        total_validated = 0
        total_failed = 0

        for source in sources:
            try:
                proxies = await grabber.extract_proxies(source)
                proxies_data = []
                for p in proxies:
                    data = p.model_dump() if hasattr(p, "model_dump") else p.__dict__
                    proxies_data.append(
                        {
                            "url": f"{data.get('protocol', 'http')}://{data.get('ip')}:{data.get('port')}",
                            "protocol": data.get("protocol", "http"),
                            "ip": data.get("ip"),
                            "port": data.get("port"),
                            "country_code": data.get("country_code"),
                            "country_name": data.get("country_name"),
                            "city": data.get("city"),
                            "latency_ms": data.get("latency_ms"),
                            "speed_mbps": data.get("speed_mbps"),
                            "anonymity": data.get("anonymity"),
                            "proxy_type": data.get("proxy_type"),
                        }
                    )
                added = await db_storage.add_proxies(session, proxies_data)

                validation_results = await db_storage.validate_and_update_proxies(
                    session, limit=min(added, 50)
                )

                total_scraped += len(proxies)
                total_added += added
                total_validated += validation_results["validated"]
                total_failed += validation_results["failed"]
                results.append(
                    {
                        "url": str(source.url),
                        "status": "success",
                        "scraped": len(proxies),
                        "added": added,
                        "validated": validation_results["validated"],
                        "failed": validation_results["failed"],
                    }
                )
            except Exception as e:
                results.append(
                    {
                        "url": str(source.url),
                        "status": "failed",
                        "error": str(e),
                        "scraped": 0,
                        "added": 0,
                        "validated": 0,
                        "failed": 0,
                    }
                )

        return {
            "message": f"Scraped {len(sources)} sources",
            "total_scraped": total_scraped,
            "total_added": total_added,
            "total_validated": total_validated,
            "total_failed": total_failed,
            "total_stored": await db_storage.count_proxies(session),
            "results": results,
        }


if __name__ == "__main__":
    import uvicorn

    uvicorn.run(app, host="0.0.0.0", port=8000)