fix( ): add 5 scrapers, fix README links

README FIX:
- Converted footer links to clickable markdown format
- Added PHASE_ROADMAP.md link

SCRAPERS RESTORED:
- scrapers/loksabha_scraper.py
- scrapers/sebi_scraper.py
- scrapers/electoral_bond_scraper.py
- scrapers/opensanctions_scraper.py
- scrapers/wikidata_scraper.py

.env.example

@@ -1,17 +1,22 @@
-# Copy this file to .env and fill in your values
-# NEVER commit .env to GitHub
+# BharatGraph Environment Variables
+# Copy this file to .env and add your REAL values there
+# This file uses placeholders only — NEVER put real keys here
+# .env is git-ignored and stays only on your computer
 
-NEO4J_URI=neo4j+s://1ahvhg668.databases.neo4j.io
-NEO4J_USER=1yuy33b8
-NEO4J_PASSWORD=t3TxyBvDeGDXyuytyjbjjkCD8P6dwAj-XlHgVBBH4SOUxcdNcCrHI
+# data.gov.in — register free at https://data.gov.in/user/register
+DATAGOV_API_KEY=your_datagov_api_key_here
 
-DATAGOV_API_KEY=579b464db66ec23gjhhg09975e9fb96f49ghfhjkvxc4bebd8f320
-OPENAI_API_KEY=your_openai_key_here
+# Neo4j AuraDB — free at https://neo4j.com/cloud/platform/aura-graph-database
+NEO4J_URI=neo4j+s://your-instance.databases.neo4j.io
+NEO4J_USER=neo4j
+NEO4J_PASSWORD=your_generated_password_here
 
-# Optional: News API (free at https://newsapi.org)
-NEWSAPI_KEY=your_newsapi_key_here
-
-# OpenSanctions API (free at opensanctions.org)
-OPENSANCTIONS_API_KEY=d86bbf9c96a76465cvclkjhgfd4f048860282
+# OpenSanctions — free API key at https://opensanctions.org/api
+OPENSANCTIONS_API_KEY=your_opensanctions_api_key_here
 
+# Copernicus Satellite — free at https://dataspace.copernicus.eu
+COPERNICUS_USER=your_copernicus_username_here
+COPERNICUS_PASSWORD=your_copernicus_password_here
 
+# News API — optional, free at https://newsapi.org
+NEWSAPI_KEY=your_newsapi_key_here

README.md

@@ -303,14 +303,9 @@ Pipeline       47 records processed and saved in 15 seconds
 
 ---
 
-## License
+## Reference Documents
 
-MIT. See `LICENSE`.
-
-## Contributing
-
-See `CONTRIBUTING.md`.
-
-## Security
-
-See `SECURITY.md`.
+- [Contributing Guidelines](CONTRIBUTING.md)
+- [Security Policy](SECURITY.md)
+- [Phase Roadmap](PHASE_ROADMAP.md)
+- [MIT License](LICENSE)

scrapers/electoral_bond_scraper.py

@@ -0,0 +1,72 @@
+import os
+from datetime import datetime
+from bs4 import BeautifulSoup
+from scrapers.base_scraper import BaseScraper
+from loguru import logger
+
+
+class ElectoralBondScraper(BaseScraper):
+
+    ECI_URL = "https://www.eci.gov.in/electoral-bonds"
+
+    def __init__(self):
+        super().__init__(name="electoral_bond", delay=2.0)
+
+    def fetch_bond_data(self, save: bool = True) -> list:
+        logger.info("[ElectoralBond] Fetching bond disclosure data...")
+        html  = self.get_html(self.ECI_URL)
+        bonds = []
+        if html:
+            bonds = self._parse_bond_page(html)
+        if not bonds:
+            logger.warning("[ElectoralBond] Live page returned no data — using sample")
+            bonds = self._get_sample_bonds()
+        if save and bonds:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filepath  = f"data/samples/electoral_bonds_{timestamp}.json"
+            self.save_json(bonds, filepath)
+            logger.success(f"[ElectoralBond] Saved {len(bonds)} records")
+        return bonds
+
+    def _parse_bond_page(self, html: str) -> list:
+        soup  = BeautifulSoup(html, "lxml")
+        bonds = []
+        for link in soup.find_all("a", href=True):
+            title = link.get_text(strip=True)
+            href  = link.get("href", "")
+            if any(kw in title.lower() for kw in ["bond","donor","purchase","redemption"]):
+                bonds.append({
+                    "title":       title,
+                    "url":         href if href.startswith("http") else self.ECI_URL,
+                    "source":      "Election Commission of India",
+                    "source_url":  self.ECI_URL,
+                    "scraped_at":  datetime.now().isoformat(),
+                    "entity_type": "electoral_bond_disclosure",
+                })
+        return bonds
+
+    def _get_sample_bonds(self) -> list:
+        return [
+            {
+                "bond_number":      "EB001",
+                "purchaser_name":   "SAMPLE CORPORATION LTD",
+                "denomination_crore": 1.0,
+                "purchase_date":    "2023-04-05",
+                "redemption_date":  "2023-04-12",
+                "redeemed_by":      "SAMPLE POLITICAL PARTY",
+                "source":           "Supreme Court ordered ECI disclosure (sample)",
+                "source_url":       self.ECI_URL,
+                "scraped_at":       datetime.now().isoformat(),
+                "entity_type":      "electoral_bond",
+            },
+        ]
+
+
+if __name__ == "__main__":
+    print("=" * 55)
+    print("BharatGraph - Electoral Bond Scraper")
+    print("=" * 55)
+    scraper = ElectoralBondScraper()
+    bonds   = scraper.fetch_bond_data(save=True)
+    print(f"\n  Total records: {len(bonds)}")
+    print("\nDone!")

scrapers/loksabha_scraper.py

@@ -0,0 +1,112 @@
+import os
+from datetime import datetime
+from bs4 import BeautifulSoup
+from scrapers.base_scraper import BaseScraper
+from loguru import logger
+
+
+class LokSabhaScraper(BaseScraper):
+
+    BASE_URL      = "https://loksabha.nic.in"
+    QUESTIONS_URL = "https://loksabha.nic.in/Questions/QResult15.aspx"
+
+    MINISTRIES = [
+        "Ministry of Finance",
+        "Ministry of Home Affairs",
+        "Ministry of Rural Development",
+        "Ministry of Health",
+    ]
+
+    def __init__(self):
+        super().__init__(name="loksabha", delay=3.0)
+        self.session.headers.update({
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36"
+            ),
+        })
+
+    def fetch_questions(self, ministry: str = "Ministry of Finance",
+                        limit: int = 20) -> list:
+        logger.info(f"[LokSabha] Fetching questions for: {ministry}")
+        html = self.get_html(self.QUESTIONS_URL, params={"ministry": ministry})
+        if not html:
+            logger.warning("[LokSabha] Could not fetch page — using sample")
+            return self._get_sample_questions(ministry)
+        questions = self._parse_questions(html, ministry, limit)
+        if not questions:
+            return self._get_sample_questions(ministry)
+        return questions
+
+    def _parse_questions(self, html: str, ministry: str, limit: int) -> list:
+        soup      = BeautifulSoup(html, "lxml")
+        questions = []
+        for row in soup.find_all("tr")[1:limit + 1]:
+            cols = row.find_all("td")
+            if len(cols) < 3:
+                continue
+            question = {
+                "question_number": cols[0].get_text(strip=True),
+                "subject":         cols[1].get_text(strip=True),
+                "member_name":     cols[2].get_text(strip=True) if len(cols) > 2 else "",
+                "ministry":        ministry,
+                "session":         "17th Lok Sabha",
+                "source":          "Lok Sabha",
+                "source_url":      self.BASE_URL,
+                "scraped_at":      datetime.now().isoformat(),
+                "entity_type":     "parliamentary_question",
+            }
+            if question["subject"]:
+                questions.append(question)
+        logger.info(f"[LokSabha] Parsed {len(questions)} questions")
+        return questions
+
+    def _get_sample_questions(self, ministry: str) -> list:
+        return [
+            {
+                "question_number": "Q001",
+                "subject":         "Status of PM-KISAN scheme beneficiary verification",
+                "member_name":     "SAMPLE MP",
+                "ministry":        ministry,
+                "session":         "17th Lok Sabha",
+                "source":          "Lok Sabha (sample)",
+                "source_url":      self.BASE_URL,
+                "scraped_at":      datetime.now().isoformat(),
+                "entity_type":     "parliamentary_question",
+            },
+            {
+                "question_number": "Q002",
+                "subject":         "Irregularities in MGNREGA fund disbursement",
+                "member_name":     "SAMPLE MP",
+                "ministry":        ministry,
+                "session":         "17th Lok Sabha",
+                "source":          "Lok Sabha (sample)",
+                "source_url":      self.BASE_URL,
+                "scraped_at":      datetime.now().isoformat(),
+                "entity_type":     "parliamentary_question",
+            },
+        ]
+
+    def fetch_all_ministries(self, save: bool = True) -> list:
+        all_questions = []
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        for ministry in self.MINISTRIES:
+            questions = self.fetch_questions(ministry, limit=10)
+            all_questions.extend(questions)
+        if save and all_questions:
+            filepath = f"data/samples/loksabha_questions_{timestamp}.json"
+            self.save_json(all_questions, filepath)
+            logger.success(f"[LokSabha] Saved {len(all_questions)} questions")
+        return all_questions
+
+
+if __name__ == "__main__":
+    print("=" * 55)
+    print("BharatGraph - Lok Sabha Scraper")
+    print("=" * 55)
+    scraper   = LokSabhaScraper()
+    questions = scraper.fetch_all_ministries(save=True)
+    print(f"\n  Total: {len(questions)}")
+    if questions:
+        print(f"  Example: {questions[0]['subject'][:60]}")
+    print("\nDone!")

scrapers/opensanctions_scraper.py

@@ -0,0 +1,72 @@
+import os
+from datetime import datetime
+from scrapers.base_scraper import BaseScraper
+from loguru import logger
+
+
+class OpenSanctionsScraper(BaseScraper):
+
+    API_URL = "https://api.opensanctions.org/search/default"
+
+    def __init__(self):
+        super().__init__(name="opensanctions", delay=1.5)
+        self.api_key = os.getenv("OPENSANCTIONS_API_KEY", "")
+        if self.api_key:
+            self.session.headers.update({"Authorization": f"ApiKey {self.api_key}"})
+        else:
+            logger.warning("[OpenSanctions] No API key — add OPENSANCTIONS_API_KEY to .env")
+
+    def search_entity(self, name: str, schema: str = "Person") -> list:
+        logger.info(f"[OpenSanctions] Searching: '{name}'")
+        params = {"q": name, "schema": schema, "limit": 10}
+        data   = self.get_json(self.API_URL, params=params)
+        if not data or "results" not in data:
+            return []
+        results = []
+        for r in data["results"]:
+            props = r.get("properties", {})
+            results.append({
+                "entity_id":    r.get("id", ""),
+                "name":         r.get("caption", name),
+                "schema":       r.get("schema", schema),
+                "datasets":     r.get("datasets", []),
+                "is_pep":       "pep" in str(r.get("datasets", [])).lower(),
+                "is_sanctioned":any(d in ["us_ofac_sdn","un_sc_sanctions","eu_fsf"]
+                                    for d in r.get("datasets", [])),
+                "countries":    props.get("country", []),
+                "positions":    props.get("position", []),
+                "source":       "OpenSanctions",
+                "source_url":   "https://opensanctions.org",
+                "scraped_at":   datetime.now().isoformat(),
+                "entity_type":  "sanctions_check",
+            })
+        logger.success(f"[OpenSanctions] Found {len(results)} matches for '{name}'")
+        return results
+
+    def screen_entity_list(self, names: list) -> dict:
+        results = {}
+        for name in names:
+            matches = self.search_entity(name)
+            if any(m["is_sanctioned"] or m["is_pep"] for m in matches):
+                results[name] = matches
+                logger.warning(f"[OpenSanctions] FLAG: '{name}' in sanctions/PEP data")
+            else:
+                logger.info(f"[OpenSanctions] CLEAR: '{name}'")
+        return results
+
+    def fetch_and_save(self, names: list, save: bool = True) -> dict:
+        results = self.screen_entity_list(names)
+        if save and results:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            self.save_json(results, f"data/samples/opensanctions_{timestamp}.json")
+        return results
+
+
+if __name__ == "__main__":
+    print("=" * 55)
+    print("BharatGraph - OpenSanctions Scraper")
+    print("=" * 55)
+    scraper = OpenSanctionsScraper()
+    results = scraper.fetch_and_save(["Sample Person", "Test Corporation"], save=True)
+    print(f"\n  Flagged: {len(results)}")
+    print("\nDone!")

scrapers/sebi_scraper.py

@@ -0,0 +1,91 @@
+import os
+from datetime import datetime
+from bs4 import BeautifulSoup
+from scrapers.base_scraper import BaseScraper
+from loguru import logger
+
+
+class SEBIScraper(BaseScraper):
+
+    BASE_URL   = "https://www.sebi.gov.in"
+    ORDERS_URL = "https://www.sebi.gov.in/enforcement/orders"
+
+    def __init__(self):
+        super().__init__(name="sebi", delay=2.0)
+        self.session.headers.update({
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36"
+            ),
+        })
+
+    def fetch_enforcement_orders(self, limit: int = 20) -> list:
+        logger.info("[SEBI] Fetching enforcement orders...")
+        html = self.get_html(self.ORDERS_URL)
+        if not html:
+            logger.warning("[SEBI] Could not fetch page — using sample")
+            return self._get_sample_orders()
+        orders = self._parse_orders(html, limit)
+        if not orders:
+            return self._get_sample_orders()
+        return orders
+
+    def _parse_orders(self, html: str, limit: int) -> list:
+        soup   = BeautifulSoup(html, "lxml")
+        orders = []
+        for link in soup.find_all("a", href=True):
+            href  = link.get("href", "")
+            title = link.get_text(strip=True)
+            if not title or len(title) < 10:
+                continue
+            if not any(kw in href.lower() for kw in ["order","adjudication","settlement"]):
+                continue
+            full_url = href if href.startswith("http") else self.BASE_URL + href
+            orders.append({
+                "title":       title,
+                "url":         full_url,
+                "order_type":  "Enforcement Order",
+                "source":      "SEBI",
+                "source_url":  self.BASE_URL,
+                "scraped_at":  datetime.now().isoformat(),
+                "entity_type": "regulatory_order",
+            })
+            if len(orders) >= limit:
+                break
+        logger.info(f"[SEBI] Parsed {len(orders)} orders")
+        return orders
+
+    def _get_sample_orders(self) -> list:
+        return [
+            {
+                "title":       "Adjudication Order against Director for insider trading",
+                "url":         "https://www.sebi.gov.in/enforcement/orders/sample-1",
+                "order_type":  "Adjudication Order",
+                "violation":   "Insider trading under SEBI (PIT) Regulations 2015",
+                "source":      "SEBI (sample)",
+                "source_url":  self.BASE_URL,
+                "scraped_at":  datetime.now().isoformat(),
+                "entity_type": "regulatory_order",
+            },
+        ]
+
+    def fetch_and_save(self, save: bool = True) -> list:
+        orders = self.fetch_enforcement_orders(limit=30)
+        if save and orders:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filepath  = f"data/samples/sebi_orders_{timestamp}.json"
+            self.save_json(orders, filepath)
+            logger.success(f"[SEBI] Saved {len(orders)} orders")
+        return orders
+
+
+if __name__ == "__main__":
+    print("=" * 55)
+    print("BharatGraph - SEBI Scraper")
+    print("=" * 55)
+    scraper = SEBIScraper()
+    orders  = scraper.fetch_and_save(save=True)
+    print(f"\n  Total orders: {len(orders)}")
+    if orders:
+        print(f"  Example: {orders[0]['title'][:60]}")
+    print("\nDone!")

scrapers/wikidata_scraper.py

@@ -0,0 +1,74 @@
+import os
+from datetime import datetime
+from scrapers.base_scraper import BaseScraper
+from loguru import logger
+
+
+class WikidataScraper(BaseScraper):
+
+    SPARQL_URL = "https://query.wikidata.org/sparql"
+
+    def __init__(self):
+        super().__init__(name="wikidata", delay=1.0)
+        self.session.headers.update({
+            "Accept":     "application/sparql-results+json",
+            "User-Agent": "BharatGraph/0.1 (civic-transparency-platform)",
+        })
+
+    def enrich_politician(self, name: str) -> dict:
+        logger.info(f"[Wikidata] Enriching: '{name}'")
+        query = f"""
+        SELECT ?person ?birthDate ?educationLabel ?positionLabel ?partyLabel WHERE {{
+          ?person wdt:P31 wd:Q5 .
+          ?person rdfs:label "{name}"@en .
+          OPTIONAL {{ ?person wdt:P569 ?birthDate }}
+          OPTIONAL {{ ?person wdt:P69 ?education }}
+          OPTIONAL {{ ?person wdt:P39 ?position }}
+          OPTIONAL {{ ?person wdt:P102 ?party }}
+          SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en" . }}
+        }}
+        LIMIT 5
+        """
+        data     = self.get_json(self.SPARQL_URL, params={"query": query, "format": "json"})
+        bindings = data.get("results",{}).get("bindings",[]) if data else []
+        if not bindings:
+            return self._empty_enrichment(name)
+        return {
+            "name":       name,
+            "wikidata_id":bindings[0].get("person",{}).get("value","").split("/")[-1],
+            "birth_date": bindings[0].get("birthDate",{}).get("value",""),
+            "education":  list({b.get("educationLabel",{}).get("value","")
+                                for b in bindings if b.get("educationLabel")}),
+            "positions":  list({b.get("positionLabel",{}).get("value","")
+                                for b in bindings if b.get("positionLabel")}),
+            "party":      bindings[0].get("partyLabel",{}).get("value",""),
+            "source":     "Wikidata",
+            "source_url": "https://www.wikidata.org",
+            "scraped_at": datetime.now().isoformat(),
+        }
+
+    def _empty_enrichment(self, name: str) -> dict:
+        return {"name": name, "wikidata_id": "", "birth_date": "",
+                "education": [], "positions": [], "party": "",
+                "source": "Wikidata", "source_url": "https://www.wikidata.org",
+                "scraped_at": datetime.now().isoformat()}
+
+    def enrich_entity_list(self, names: list, save: bool = True) -> list:
+        results = [self.enrich_politician(n) for n in names]
+        if save and results:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            self.save_json(results, f"data/samples/wikidata_enrichment_{timestamp}.json")
+            logger.success(f"[Wikidata] Saved enrichment for {len(results)} entities")
+        return results
+
+
+if __name__ == "__main__":
+    print("=" * 55)
+    print("BharatGraph - Wikidata Scraper")
+    print("=" * 55)
+    scraper = WikidataScraper()
+    results = scraper.enrich_entity_list(["Narendra Modi", "Rahul Gandhi"], save=True)
+    for r in results:
+        print(f"\n  {r['name']}: ID={r['wikidata_id']} party={r['party']}")
+        print(f"  positions={r['positions'][:2]}")
+    print("\nDone!")