api/routes/sources.py

import os, sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))

from datetime import datetime
from fastapi import APIRouter, Depends
from loguru import logger

from api.dependencies import get_db

router = APIRouter()

SCRAPER_META = {
    "myneta":          {"name":"MyNeta / ECI",              "url":"https://myneta.info",        "data":"Politicians, assets, criminal cases"},
    "mca":             {"name":"MCA21",                     "url":"https://www.mca.gov.in",     "data":"Companies, directors, CIN"},
    "gem":             {"name":"GeM",                       "url":"https://gem.gov.in",         "data":"Government contracts"},
    "cag":             {"name":"CAG",                       "url":"https://cag.gov.in",         "data":"Audit reports, irregularities"},
    "pib":             {"name":"PIB",                       "url":"https://pib.gov.in",         "data":"Press releases"},
    "datagov":         {"name":"data.gov.in",               "url":"https://data.gov.in",        "data":"MGNREGA, PM-KISAN, open data"},
    "loksabha":        {"name":"Lok Sabha",                 "url":"https://loksabha.nic.in",    "data":"Parliamentary questions, debates"},
    "sebi":            {"name":"SEBI",                      "url":"https://www.sebi.gov.in",    "data":"Enforcement orders"},
    "electoral_bond":  {"name":"Electoral Bonds (ECI/SC)",  "url":"https://eci.gov.in",         "data":"Donor-party data"},
    "opensanctions":   {"name":"OpenSanctions",             "url":"https://opensanctions.org",  "data":"Global PEP and sanctions"},
    "icij":            {"name":"ICIJ Offshore Leaks",       "url":"https://offshoreleaks.icij.org","data":"Panama, Pandora, Paradise Papers"},
    "wikidata":        {"name":"Wikidata",                  "url":"https://www.wikidata.org",   "data":"Entity enrichment, career data"},
    "njdg":            {"name":"NJDG / eCourts",            "url":"https://njdg.ecourts.gov.in","data":"Court cases, judgments"},
    "ed":              {"name":"Enforcement Directorate",   "url":"https://enforcementdirectorate.gov.in","data":"PMLA press releases"},
    "cvc":             {"name":"CVC",                       "url":"https://cvc.gov.in",         "data":"Vigilance advisories"},
    "ncrb":            {"name":"NCRB",                      "url":"https://ncrb.gov.in",        "data":"Crime statistics"},
    "lgd":             {"name":"LGD / Panchayati Raj",      "url":"https://lgdirectory.gov.in", "data":"Local government data"},
    "ibbi":            {"name":"IBBI",                      "url":"https://ibbi.gov.in",        "data":"Corporate insolvency records"},
    "ngo_darpan":      {"name":"NGO Darpan / NITI Aayog",   "url":"https://ngodarpan.gov.in",   "data":"NGO registration and funding"},
    "cppp":            {"name":"CPPP",                      "url":"https://eprocure.gov.in",    "data":"Procurement transparency"},
}


@router.get("/sources")
def sources_summary(driver=Depends(get_db)):
    """Returns count of nodes per source dataset loaded in Neo4j."""
    logger.info("[Sources] Summary requested")

    db_counts = {}
    try:
        with driver.session() as s:
            rows = s.run(
                """
                MATCH (n)
                WHERE n.source IS NOT NULL OR n.dataset IS NOT NULL
                RETURN coalesce(n.dataset, n.source) AS source, count(*) AS total
                ORDER BY total DESC
                """
            ).data()
            db_counts = {r["source"]: r["total"] for r in rows}
    except Exception as e:
        logger.warning(f"[Sources] DB query failed: {e}")

    label_counts = {}
    try:
        with driver.session() as s:
            rows = s.run(
                """
                MATCH (n)
                RETURN labels(n)[0] AS label, count(*) AS total
                ORDER BY total DESC
                """
            ).data()
            label_counts = {r["label"]: r["total"] for r in rows if r.get("label")}
    except Exception as e:
        logger.warning(f"[Sources] Label count failed: {e}")

    sources = []
    for key, meta in SCRAPER_META.items():
        count = db_counts.get(key, 0)
        sources.append({
            "dataset":      key,
            "name":         meta["name"],
            "url":          meta["url"],
            "data_type":    meta["data"],
            "records_in_db": count,
            "status":       "loaded" if count > 0 else "pending",
        })

    sources.sort(key=lambda x: -x["records_in_db"])

    total_records = sum(label_counts.values()) if label_counts else 0

    return {
        "total_records":  total_records,
        "label_counts":   label_counts,
        "source_count":   len(SCRAPER_META),
        "loaded_sources": sum(1 for s in sources if s["status"] == "loaded"),
        "sources":        sources,
        "generated_at":   datetime.now().isoformat(),
    }


@router.get("/sources/{dataset}")
def source_detail(dataset: str, driver=Depends(get_db)):
    """Returns sample records from a specific dataset."""
    logger.info(f"[Sources] Detail for {dataset}")

    meta = SCRAPER_META.get(dataset)
    if not meta:
        return {"error": f"Unknown dataset: {dataset}",
                "available": list(SCRAPER_META.keys())}

    records = []
    try:
        with driver.session() as s:
            rows = s.run(
                """
                MATCH (n)
                WHERE n.source = $src
                RETURN n.id AS id, labels(n)[0] AS label,
                       coalesce(n.name, n.title) AS name,
                       n.state AS state, n.scraped_at AS scraped_at
                LIMIT 20
                """,
                src=dataset
            ).data()
            records = [dict(r) for r in rows]
    except Exception as e:
        logger.warning(f"[Sources] Detail query failed: {e}")

    return {
        "dataset":      dataset,
        "name":         meta["name"],
        "url":          meta["url"],
        "data_type":    meta["data"],
        "sample_count": len(records),
        "records":      records,
    }